Skip to content

Commit 16b038f

Browse files
mriggermrigger
committed
[Clickhouse] Add a template for Clickhouse
1 parent 0814b6c commit 16b038f

9 files changed

Lines changed: 447 additions & 5 deletions

‎src/sqlancer/Main.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.beust.jcommander.JCommander;
2828
import com.beust.jcommander.JCommander.Builder;
2929

30+
import sqlancer.clickhouse.ClickhouseProvider;
3031
import sqlancer.cockroachdb.CockroachDBProvider;
3132
import sqlancer.mariadb.MariaDBProvider;
3233
import sqlancer.mysql.MySQLProvider;
@@ -294,6 +295,7 @@ public static void main(String[] args) {
294295
providers.add(new TiDBProvider());
295296
providers.add(new PostgresProvider());
296297
providers.add(new TDEngineProvider());
298+
providers.add(new ClickhouseProvider());
297299
Map<String, DatabaseProvider<?, ?>> nameToProvider = new HashMap<>();
298300
Map<String, Object> nameToOptions = new HashMap<>();
299301
MainOptions options = new MainOptions();

‎src/sqlancer/StateToReproduce.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
import sqlancer.tdengine.expr.TDEngineExpression;
1919
import sqlancer.tdengine.expr.TDEngineSelectStatement;
2020

21-
public abstract class StateToReproduce {
21+
public class StateToReproduce {
2222

2323
public final List<Query> statements = new ArrayList<>();
2424
public String queryString;

‎src/sqlancer/clickhouse/ClickhouseOptions.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package sqlancer.clickhouse;
2+
3+
import com.beust.jcommander.Parameters;
4+
5+
@Parameters
6+
public class ClickhouseOptions {
7+
8+
}

‎src/sqlancer/clickhouse/ClickhouseProvider.java‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package sqlancer.clickhouse;
2+
3+
import java.io.IOException;
4+
import java.sql.Connection;
5+
import java.sql.DriverManager;
6+
import java.sql.SQLException;
7+
import java.sql.Statement;
8+
9+
import sqlancer.AbstractAction;
10+
import sqlancer.DatabaseProvider;
11+
import sqlancer.GlobalState;
12+
import sqlancer.IgnoreMeException;
13+
import sqlancer.Main.QueryManager;
14+
import sqlancer.Main.StateLogger;
15+
import sqlancer.Query;
16+
import sqlancer.QueryAdapter;
17+
import sqlancer.QueryProvider;
18+
import sqlancer.Randomly;
19+
import sqlancer.StateToReproduce;
20+
import sqlancer.StatementExecutor;
21+
import sqlancer.clickhouse.ClickhouseProvider.ClickhouseGlobalState;
22+
import sqlancer.clickhouse.gen.ClickhouseInsertGenerator;
23+
import sqlancer.clickhouse.gen.ClickhouseTableGenerator;
24+
25+
public class ClickhouseProvider implements DatabaseProvider<ClickhouseGlobalState, ClickhouseOptions> {
26+
27+
public static enum Action implements AbstractAction<ClickhouseGlobalState> {
28+
29+
INSERT(ClickhouseInsertGenerator::getQuery);
30+
31+
private final QueryProvider<ClickhouseGlobalState> queryProvider;
32+
33+
private Action(QueryProvider<ClickhouseGlobalState> queryProvider) {
34+
this.queryProvider = queryProvider;
35+
}
36+
37+
public Query getQuery(ClickhouseGlobalState state) throws SQLException {
38+
return queryProvider.getQuery(state);
39+
}
40+
}
41+
42+
private static int mapActions(ClickhouseGlobalState globalState, Action a) {
43+
Randomly r = globalState.getRandomly();
44+
switch (a) {
45+
case INSERT:
46+
return r.getInteger(0, globalState.getOptions().getMaxNumberInserts());
47+
default:
48+
throw new AssertionError(a);
49+
}
50+
}
51+
52+
public static class ClickhouseGlobalState extends GlobalState<ClickhouseOptions> {
53+
54+
private ClickhouseSchema schema;
55+
56+
public void setSchema(ClickhouseSchema schema) {
57+
this.schema = schema;
58+
}
59+
60+
public ClickhouseSchema getSchema() {
61+
return schema;
62+
}
63+
64+
}
65+
66+
@Override
67+
public void generateAndTestDatabase(ClickhouseGlobalState globalState) throws SQLException {
68+
StateLogger logger = globalState.getLogger();
69+
QueryManager manager = globalState.getManager();
70+
globalState.setSchema(
71+
ClickhouseSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName()));
72+
for (int i = 0; i < Randomly.fromOptions(1); i++) {
73+
boolean success = false;
74+
do {
75+
Query qt = new ClickhouseTableGenerator().getQuery(globalState);
76+
success = manager.execute(qt);
77+
logger.writeCurrent(globalState.getState());
78+
globalState.setSchema(
79+
ClickhouseSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName()));
80+
try {
81+
logger.getCurrentFileWriter().close();
82+
} catch (IOException e) {
83+
// TODO Auto-generated catch block
84+
e.printStackTrace();
85+
}
86+
logger.currentFileWriter = null;
87+
} while (!success);
88+
}
89+
90+
StatementExecutor<ClickhouseGlobalState, Action> se = new StatementExecutor<ClickhouseGlobalState, Action>(
91+
globalState, globalState.getDatabaseName(), Action.values(), ClickhouseProvider::mapActions, (q) -> {
92+
if (q.couldAffectSchema()) {
93+
globalState.setSchema(ClickhouseSchema.fromConnection(globalState.getConnection(),
94+
globalState.getDatabaseName()));
95+
}
96+
if (globalState.getSchema().getDatabaseTables().isEmpty()) {
97+
throw new IgnoreMeException();
98+
}
99+
});
100+
se.executeStatements();
101+
manager.incrementCreateDatabase();
102+
}
103+
104+
@Override
105+
public Connection createDatabase(String databaseName, StateToReproduce state) throws SQLException {
106+
String url = "jdbc:clickhouse://localhost:8123/test";
107+
Connection con = DriverManager.getConnection(url, "", "password");
108+
state.statements.add(new QueryAdapter("USE test"));
109+
state.statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"));
110+
String createDatabaseCommand = "CREATE DATABASE " + databaseName;
111+
state.statements.add(new QueryAdapter(createDatabaseCommand));
112+
state.statements.add(new QueryAdapter("USE " + databaseName));
113+
try (Statement s = con.createStatement()) {
114+
s.execute("DROP DATABASE IF EXISTS " + databaseName);
115+
}
116+
try (Statement s = con.createStatement()) {
117+
s.execute(createDatabaseCommand);
118+
}
119+
con.close();
120+
con = DriverManager.getConnection("jdbc:clickhouse://localhost:8123/" + databaseName, "", "password");
121+
return con;
122+
}
123+
124+
@Override
125+
public String getDBMSName() {
126+
return "Clickhouse";
127+
}
128+
129+
@Override
130+
public StateToReproduce getStateToReproduce(String databaseName) {
131+
return new StateToReproduce(databaseName);
132+
}
133+
134+
@Override
135+
public ClickhouseGlobalState generateGlobalState() {
136+
return new ClickhouseGlobalState();
137+
}
138+
139+
@Override
140+
public ClickhouseOptions getCommand() {
141+
return new ClickhouseOptions();
142+
}
143+
}

‎src/sqlancer/clickhouse/ClickhouseSchema.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
package sqlancer.clickhouse;
2+
3+
import java.sql.Connection;
4+
import java.sql.ResultSet;
5+
import java.sql.SQLException;
6+
import java.sql.Statement;
7+
import java.util.ArrayList;
8+
import java.util.Collections;
9+
import java.util.List;
10+
11+
import sqlancer.Randomly;
12+
import sqlancer.clickhouse.ClickhouseSchema.ClickhouseTable;
13+
import sqlancer.schema.AbstractSchema;
14+
import sqlancer.schema.AbstractTable;
15+
import sqlancer.schema.AbstractTableColumn;
16+
import sqlancer.schema.AbstractTables;
17+
import sqlancer.schema.TableIndex;
18+
19+
public class ClickhouseSchema extends AbstractSchema<ClickhouseTable> {
20+
21+
public static enum ClickhouseDataType {
22+
23+
INT;
24+
25+
public static ClickhouseDataType getRandom() {
26+
return Randomly.fromOptions(values());
27+
}
28+
29+
}
30+
31+
public static class ClickhouseCompositeDataType {
32+
33+
private final ClickhouseDataType dataType;
34+
35+
private final int size;
36+
37+
public ClickhouseCompositeDataType(ClickhouseDataType dataType) {
38+
this.dataType = dataType;
39+
this.size = -1;
40+
}
41+
42+
public ClickhouseCompositeDataType(ClickhouseDataType dataType, int size) {
43+
this.dataType = dataType;
44+
this.size = size;
45+
}
46+
47+
public ClickhouseDataType getPrimitiveDataType() {
48+
return dataType;
49+
}
50+
51+
public int getSize() {
52+
if (size == -1) {
53+
throw new AssertionError(this);
54+
}
55+
return size;
56+
}
57+
58+
public static ClickhouseCompositeDataType getInt(int size) {
59+
return new ClickhouseCompositeDataType(ClickhouseDataType.INT, size);
60+
}
61+
62+
}
63+
64+
public static class ClickhouseColumn extends AbstractTableColumn<ClickhouseTable, ClickhouseCompositeDataType> {
65+
66+
public ClickhouseColumn(String name, ClickhouseCompositeDataType columnType) {
67+
super(name, null, columnType);
68+
}
69+
70+
}
71+
72+
public static class ClickhouseTables extends AbstractTables<ClickhouseTable, ClickhouseColumn> {
73+
74+
public ClickhouseTables(List<ClickhouseTable> tables) {
75+
super(tables);
76+
}
77+
78+
}
79+
80+
public ClickhouseSchema(List<ClickhouseTable> databaseTables) {
81+
super(databaseTables);
82+
}
83+
84+
public ClickhouseTables getRandomTableNonEmptyTables() {
85+
return new ClickhouseTables(Randomly.nonEmptySubset(getDatabaseTables()));
86+
}
87+
88+
private static ClickhouseCompositeDataType getColumnType(String typeString) {
89+
ClickhouseDataType primitiveType;
90+
switch (typeString) {
91+
case "Int32":
92+
primitiveType = ClickhouseDataType.INT;
93+
break;
94+
default:
95+
throw new AssertionError(typeString);
96+
}
97+
return new ClickhouseCompositeDataType(primitiveType);
98+
}
99+
100+
public static class ClickhouseTable extends AbstractTable<ClickhouseColumn, TableIndex> {
101+
102+
public ClickhouseTable(String tableName, List<ClickhouseColumn> columns, List<TableIndex> indexes, boolean isView) {
103+
super(tableName, columns, indexes, isView);
104+
}
105+
106+
}
107+
108+
public static ClickhouseSchema fromConnection(Connection con, String databaseName) throws SQLException {
109+
List<ClickhouseTable> databaseTables = new ArrayList<>();
110+
List<String> tableNames = getTableNames(con);
111+
for (String tableName : tableNames) {
112+
List<ClickhouseColumn> databaseColumns = getTableColumns(con, tableName);
113+
List<TableIndex> indexes = Collections.emptyList();
114+
boolean isView = tableName.startsWith("v");
115+
ClickhouseTable t = new ClickhouseTable(tableName, databaseColumns, indexes, isView);
116+
for (ClickhouseColumn c : databaseColumns) {
117+
c.setTable(t);
118+
}
119+
databaseTables.add(t);
120+
121+
}
122+
return new ClickhouseSchema(databaseTables);
123+
}
124+
125+
private static List<String> getTableNames(Connection con) throws SQLException {
126+
List<String> tableNames = new ArrayList<>();
127+
try (Statement s = con.createStatement()) {
128+
ResultSet tableRs = s.executeQuery("SHOW TABLES");
129+
while (tableRs.next()) {
130+
String tableName = tableRs.getString(1);
131+
tableNames.add(tableName);
132+
}
133+
}
134+
return tableNames;
135+
}
136+
137+
private static List<ClickhouseColumn> getTableColumns(Connection con, String tableName) throws SQLException {
138+
List<ClickhouseColumn> columns = new ArrayList<>();
139+
try (Statement s = con.createStatement()) {
140+
try (ResultSet rs = s.executeQuery("DESCRIBE " + tableName)) {
141+
while (rs.next()) {
142+
String columnName = rs.getString("name");
143+
String dataType = rs.getString("type");
144+
ClickhouseColumn c = new ClickhouseColumn(columnName, getColumnType(dataType));
145+
columns.add(c);
146+
}
147+
}
148+
}
149+
return columns;
150+
}
151+
152+
}

‎src/sqlancer/clickhouse/gen/ClickhouseInsertGenerator.java‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package sqlancer.clickhouse.gen;
2+
3+
import java.sql.SQLException;
4+
import java.util.HashSet;
5+
import java.util.List;
6+
import java.util.Set;
7+
import java.util.stream.Collectors;
8+
9+
import sqlancer.Query;
10+
import sqlancer.QueryAdapter;
11+
import sqlancer.clickhouse.ClickhouseProvider.ClickhouseGlobalState;
12+
import sqlancer.clickhouse.ClickhouseSchema.ClickhouseColumn;
13+
import sqlancer.clickhouse.ClickhouseSchema.ClickhouseTable;
14+
import sqlancer.gen.AbstractInsertGenerator;
15+
16+
public class ClickhouseInsertGenerator extends AbstractInsertGenerator<ClickhouseColumn> {
17+
18+
private ClickhouseGlobalState globalState;
19+
private Set<String> errors = new HashSet<>();
20+
21+
public ClickhouseInsertGenerator(ClickhouseGlobalState globalState) {
22+
this.globalState = globalState;
23+
}
24+
25+
public static Query getQuery(ClickhouseGlobalState globalState) throws SQLException {
26+
return new ClickhouseInsertGenerator(globalState).get();
27+
}
28+
29+
private Query get() {
30+
ClickhouseTable table = globalState.getSchema().getRandomTable(t -> !t.isView());
31+
List<ClickhouseColumn> columns = table.getRandomNonEmptyColumnSubset();
32+
sb.append("INSERT INTO ");
33+
sb.append(table.getName());
34+
sb.append("(");
35+
sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", ")));
36+
sb.append(")");
37+
sb.append(" VALUES ");
38+
insertColumns(columns);
39+
return new QueryAdapter(sb.toString(), errors);
40+
}
41+
42+
@Override
43+
protected void insertValue(ClickhouseColumn tiDBColumn) {
44+
sb.append(globalState.getRandomly().getInteger());
45+
}
46+
47+
}

0 commit comments

Comments
 (0)