diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..a217b347e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: +- package-ecosystem: maven + directory: "/" + schedule: + interval: daily + open-pull-requests-limit: 10 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..9f0dbba32 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,701 @@ +name: ci + +on: + push: + branches: + - main + paths: + - 'src/**' + - 'test/**' + - '.github/workflows/*.yml' + - 'pom.xml' + pull_request: + branches: + - main + paths: + - 'src/**' + - 'test/**' + - '.github/workflows/*.yml' + - 'pom.xml' + +jobs: + misc: + name: General tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Verify + run: mvn -B verify -DskipTests=true + - name: Misc Tests + run: mvn -Djacoco.skip=true -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Naming Convention Tests + run: python src/check_names.py + + citus: + name: DBMS Tests (Citus) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up Citus + run: | + echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + curl https://install.citusdata.com/community/deb.sh | sudo bash + sudo sed -i 's/noble/jammy/g' /etc/apt/sources.list.d/citusdata_community.list # https://github.com/citusdata/citus/issues/7692 + sudo apt-get update + sudo apt-get -y install postgresql-17-citus-13.0 + sudo chown -R $USER:$USER /var/run/postgresql + export PATH=/usr/lib/postgresql/17/bin:$PATH + cd ~ + mkdir -p citus/coordinator citus/worker1 citus/worker2 + initdb -D citus/coordinator + initdb -D citus/worker1 + initdb -D citus/worker2 + echo "shared_preload_libraries = 'citus'" >> citus/coordinator/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> citus/worker1/postgresql.conf + echo "shared_preload_libraries = 'citus'" >> citus/worker2/postgresql.conf + pg_ctl -D citus/coordinator -o "-p 9700" -l coordinator_logfile start || cat coordinator_logfile || cat citus/coordinator/coordinator_logfile + pg_ctl -D citus/worker1 -o "-p 9701" -l worker1_logfile start + ls citus/worker1 + pg_ctl -D citus/worker2 -o "-p 9702" -l worker2_logfile start + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9700 -d postgres -U $USER + createdb test -p 9700 -U $USER + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9701 -d postgres -U $USER + createdb test -p 9701 -U $USER + psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9702 -d postgres -U $USER + createdb test -p 9702 -U $USER + psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test + psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test + psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test + psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test + psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test + - name: Run Tests + run: CITUS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCitus test + + clickhouse: + name: DBMS Tests (ClickHouse) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up ClickHouse + run: | + docker pull clickhouse/clickhouse-server:24.3.1.2672 + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 + until curl -sf http://127.0.0.1:8123/ping 2>/dev/null; do sleep 1; done + - name: Run Tests + run: CLICKHOUSE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test + - name: Show fatal errors + run: docker exec clickhouse-server grep Fatal /var/log/clickhouse-server/clickhouse-server.log || echo No Fatal Errors found + - name: Teardown ClickHouse server + run: | + docker stop clickhouse-server + docker rm clickhouse-server + + cockroachdb: + name: DBMS Tests (CockroachDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up CockroachDB + run: | + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & + until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done + - name: Create SQLancer user + run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + - name: Run Tests + run: | + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBNoREC test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBTLP test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBCERT test + + cockroachdb-qpg: + name: QPG Tests (CockroachDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up CockroachDB + run: | + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & + until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done + - name: Create SQLancer user + run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + - name: Run Tests + run: COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBQPG test + + databend: + name: DBMS Tests (Databend) + runs-on: ubuntu-latest + services: + databend: + image: datafuselabs/databend:v1.2.900-nightly + env: + QUERY_DEFAULT_USER: sqlancer + QUERY_DEFAULT_PASSWORD: sqlancer + ports: + - 8000:8000 + - 3307:3307 + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendTLP test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendNoREC test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendPQS test + + datafusion: + name: DBMS Tests (DataFusion) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + workspaces: src/sqlancer/datafusion/server/datafusion_server + - name: Build DataFusion Server + run: | + cd src/sqlancer/datafusion/server/datafusion_server + cargo build + - name: Start DataFusion Server + run: | + cd src/sqlancer/datafusion/server/datafusion_server + cargo run & + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Wait for DataFusion Server + run: | + for i in $(seq 1 30); do + if nc -z 127.0.0.1 50051 2>/dev/null; then + echo "DataFusion server is ready" + exit 0 + fi + echo "Waiting for DataFusion server... ($i/30)" + sleep 10 + done + echo "DataFusion server failed to start within 300s" + exit 1 + - name: Run Tests + run: | + DATAFUSION_AVAILABLE=true mvn -Djacoco.skip=true test -Pdatafusion-tests + + duckdb: + name: DBMS Tests (DuckDB) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build + run: mvn -B package -DskipTests=true + - name: DuckDB Tests + run: | + mvn -Djacoco.skip=true -Dtest=TestDuckDBTLP test + mvn -Djacoco.skip=true -Dtest=TestDuckDBNoREC test + + h2: + name: DBMS Tests (H2) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: mvn -Djacoco.skip=true -Dtest=TestH2 test + + hive: + name: DBMS Tests (Hive) + runs-on: ubuntu-latest + services: + metastore: + image: apache/hive:4.0.1 + env: + SERVICE_NAME: 'metastore' + ports: + - 9083:9083 + volumes: + - warehouse:/opt/hive/data/warehouse + hiveserver2: + image: apache/hive:4.0.1 + env: + SERVICE_NAME: 'hiveserver2' + ports: + - 10000:10000 + - 10002:10002 + volumes: + - warehouse:/opt/hive/data/warehouse + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: HIVE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestHiveTLP test + + spark: + name: DBMS Tests (Spark) + runs-on: ubuntu-latest + + services: + spark: + image: apache/spark:3.5.1 + ports: + - 10000:10000 + + command: >- + /opt/spark/bin/spark-submit + --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 + --name "Thrift JDBC/ODBC Server" + --master local[*] + --driver-memory 4g + --conf spark.hive.server2.thrift.port=10000 + --conf spark.sql.warehouse.dir=/tmp/spark-warehouse + spark-internal + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + + - name: Build SQLancer + run: mvn -B package -DskipTests=true + + - name: Run Tests + run: SPARK_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestSparkTLP test + + hsqldb: + name: DBMS Tests (HSQLDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + mvn -Djacoco.skip=true -Dtest=TestHSQLDBNoREC test + mvn -Djacoco.skip=true -Dtest=TestHSQLDBTLP test + + mariadb: + name: DBMS Tests (MariaDB) + runs-on: ubuntu-latest + services: + mysql: + image: mariadb:11.7.2 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=10 + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Create SQLancer User + run: sudo mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" + - name: Run Tests + run: MARIADB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestMariaDB test + + materialize: + name: DBMS Tests (Materialize) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Materialize + run: | + docker pull materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest + until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeNoREC + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeTLP + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializePQS + + materialize-qpg: + name: QPG Tests (Materialize) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Materialize + run: | + docker pull materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest + until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeQPG + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeQueryPlan + + mysql: + name: DBMS Tests (MySQL, CERT creation only) + runs-on: ubuntu-latest + services: + mysql: + image: mysql:9.7.0 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10 + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Create SQLancer user + run: mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" + - name: Run Tests + run: | + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLPQS + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLTLP + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLCERT + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLDQE + + oceanbase: + name: DBMS Tests (OceanBase) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up OceanBase + run: | + docker run -p 2881:2881 --name oceanbase-ce -e MODE=mini -d oceanbase/oceanbase-ce:4.2.1-lts + until mysql -h127.1 -uroot@test -P2881 --connect-timeout=3 -Doceanbase -A -e "SELECT 1" 2>/dev/null; do sleep 5; done + mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" + - name: Run Tests + run: | + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBaseNoREC + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBasePQS + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBaseTLP + postgres: + name: DBMS Tests (PostgreSQL) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up PostgreSQL + uses: harmon758/postgresql-action@v1.0.0 + with: + postgresql version: '18' + postgresql user: 'sqlancer' + postgresql password: 'sqlancer' + postgresql db: 'test' + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresPQS test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresTLP test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresNoREC test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresCERT test + + presto: + name: DBMS Tests (Presto) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Set up Presto + run: | + docker pull prestodb/presto:latest + echo "connector.name=memory" >> memory.properties + docker run -p 8080:8080 -d -v ./memory.properties:/opt/presto-server/etc/catalog/memory.properties --name presto prestodb/presto:latest + until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + PRESTO_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPrestoNoREC test + docker restart presto && until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done + PRESTO_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPrestoTLP test + sqlite: + name: DBMS Tests (SQLite) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build + run: mvn -B package -DskipTests=true + - name: SQLite Tests + run: | + mvn -Djacoco.skip=true -Dtest=TestSQLitePQS test + mvn -Djacoco.skip=true -Dtest=TestSQLiteTLP test + mvn -Djacoco.skip=true -Dtest=TestSQLiteNoREC test + mvn -Djacoco.skip=true -Dtest=TestSQLiteCODDTest test + + sqlite-qpg: + name: QPG Tests (SQLite) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build + run: mvn -B package -DskipTests=true + - name: SQLite Tests for QPG + run: | + mvn -Djacoco.skip=true -Dtest=TestSQLiteQPG test + + tidb: + name: DBMS Tests (TiDB, TLP creation only) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up TiDB + run: | + docker pull hawkingrei/tidb-playground:nightly-2025-09-16 + docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 + until mysql -h 127.0.0.1 -P 4000 -u root --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done + - name: Create SQLancer user + run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Run Tests + run: | + TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBTLP test + TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBCERT test + + tidb-qpg: + name: QPG Tests (TiDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up TiDB + run: | + docker pull hawkingrei/tidb-playground:nightly-2025-09-16 + docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 + until mysql -h 127.0.0.1 -P 4000 -u root --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done + - name: Create SQLancer user + run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Run Tests + run: TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBQPG test + + yugabyte: + name: DBMS Tests (YugabyteDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up Yugabyte + run: | + docker pull yugabytedb/yugabyte:latest + docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false --tserver_flags="ysql_yb_enable_listen_notify=true" --master_flags="ysql_yb_enable_listen_notify=true" + until pg_isready -h localhost -p 5433; do sleep 1; done + until nc -z localhost 9042; do sleep 1; done + - name: Run Tests + run: | + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLNoREC test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLTLP test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLPQS test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYCQL test + + doris: + name: DBMS Tests (Apache Doris) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: install mysql client + run: | + sudo apt update + sudo apt install mysql-client --assume-yes + - name: Cache Apache Doris tarball + uses: actions/cache@v4 + with: + path: apache-doris-2.1.4-bin-x64.tar.gz + key: apache-doris-2.1.4-bin-x64-tarball + - name: Set up Apache Doris + run: | + sudo sysctl -w vm.max_map_count=2000000 + [ -f apache-doris-2.1.4-bin-x64.tar.gz ] || wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz + tar zxf apache-doris-2.1.4-bin-x64.tar.gz + mv apache-doris-2.1.4-bin-x64 apache-doris + sudo swapoff -a + cd apache-doris/fe + ./bin/start_fe.sh --daemon + cd ../be + ./bin/start_be.sh --daemon + + until mysql -u root -h 127.0.0.1 --port 9030 --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done + IP=$(hostname -I | awk '{print $1}') + mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';" + mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisNoREC test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisPQS test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisTLP test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..6194c1529 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,45 @@ +name: Publish package to the Maven Central Repository and Docker Hub +on: + release: + types: [created] + workflow_dispatch: +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Maven Central Repository + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - name: Install gpg secret key + run: cat <(echo -e "${{ secrets.OSSRH_GPG_SECRET_KEY }}") | gpg --batch --import + - name: Publish package + run: mvn --batch-mode deploy -DskipTests=true -DreleaseBuild=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} + env: + MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Push to Docker Hub + uses: docker/build-push-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + repository: mrigger/sqlancer + tag_with_ref: true diff --git a/.gitignore b/.gitignore index 9fd02264e..d7cbeb55f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,16 @@ target/ .classpath -.settings +.settings/ +.vscode .project .checkstyle *.DS_Store +.idea +SQLancer.iml +dependency-reduced-pom.xml +database0.db +databaseconnectiontest.db +database*.log +database*.properties +database*.script +databases/ \ No newline at end of file diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 000000000..0a659c1a7 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,114 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.codeComplete.visibilityCheck=enabled +org.eclipse.jdt.core.compiler.annotation.inheritNullAnnotations=disabled +org.eclipse.jdt.core.compiler.annotation.missingNonNullByDefaultAnnotation=ignore +org.eclipse.jdt.core.compiler.annotation.nonnull=org.eclipse.jdt.annotation.NonNull +org.eclipse.jdt.core.compiler.annotation.nonnull.secondary= +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault=org.eclipse.jdt.annotation.NonNullByDefault +org.eclipse.jdt.core.compiler.annotation.nonnullbydefault.secondary= +org.eclipse.jdt.core.compiler.annotation.nullable=org.eclipse.jdt.annotation.Nullable +org.eclipse.jdt.core.compiler.annotation.nullable.secondary= +org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate +org.eclipse.jdt.core.compiler.problem.APILeak=warning +org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=warning +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.autoboxing=ignore +org.eclipse.jdt.core.compiler.problem.comparingIdentical=warning +org.eclipse.jdt.core.compiler.problem.deadCode=warning +org.eclipse.jdt.core.compiler.problem.deprecation=warning +org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled +org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled +org.eclipse.jdt.core.compiler.problem.discouragedReference=warning +org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore +org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore +org.eclipse.jdt.core.compiler.problem.fatalOptionalError=disabled +org.eclipse.jdt.core.compiler.problem.fieldHiding=ignore +org.eclipse.jdt.core.compiler.problem.finalParameterBound=warning +org.eclipse.jdt.core.compiler.problem.finallyBlockNotCompletingNormally=warning +org.eclipse.jdt.core.compiler.problem.forbiddenReference=error +org.eclipse.jdt.core.compiler.problem.hiddenCatchBlock=warning +org.eclipse.jdt.core.compiler.problem.includeNullInfoFromAsserts=disabled +org.eclipse.jdt.core.compiler.problem.incompatibleNonInheritedInterfaceMethod=warning +org.eclipse.jdt.core.compiler.problem.incompleteEnumSwitch=warning +org.eclipse.jdt.core.compiler.problem.indirectStaticAccess=ignore +org.eclipse.jdt.core.compiler.problem.localVariableHiding=ignore +org.eclipse.jdt.core.compiler.problem.methodWithConstructorName=warning +org.eclipse.jdt.core.compiler.problem.missingDefaultCase=ignore +org.eclipse.jdt.core.compiler.problem.missingDeprecatedAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingEnumCaseDespiteDefault=disabled +org.eclipse.jdt.core.compiler.problem.missingHashCodeMethod=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotation=ignore +org.eclipse.jdt.core.compiler.problem.missingOverrideAnnotationForInterfaceMethodImplementation=enabled +org.eclipse.jdt.core.compiler.problem.missingSerialVersion=warning +org.eclipse.jdt.core.compiler.problem.missingSynchronizedOnInheritedMethod=ignore +org.eclipse.jdt.core.compiler.problem.noEffectAssignment=warning +org.eclipse.jdt.core.compiler.problem.noImplicitStringConversion=warning +org.eclipse.jdt.core.compiler.problem.nonExternalizedStringLiteral=ignore +org.eclipse.jdt.core.compiler.problem.nonnullParameterAnnotationDropped=warning +org.eclipse.jdt.core.compiler.problem.nonnullTypeVariableFromLegacyInvocation=warning +org.eclipse.jdt.core.compiler.problem.nullAnnotationInferenceConflict=error +org.eclipse.jdt.core.compiler.problem.nullReference=warning +org.eclipse.jdt.core.compiler.problem.nullSpecViolation=error +org.eclipse.jdt.core.compiler.problem.nullUncheckedConversion=warning +org.eclipse.jdt.core.compiler.problem.overridingPackageDefaultMethod=warning +org.eclipse.jdt.core.compiler.problem.parameterAssignment=ignore +org.eclipse.jdt.core.compiler.problem.pessimisticNullAnalysisForFreeTypeVariables=warning +org.eclipse.jdt.core.compiler.problem.possibleAccidentalBooleanAssignment=ignore +org.eclipse.jdt.core.compiler.problem.potentialNullReference=ignore +org.eclipse.jdt.core.compiler.problem.potentiallyUnclosedCloseable=ignore +org.eclipse.jdt.core.compiler.problem.rawTypeReference=warning +org.eclipse.jdt.core.compiler.problem.redundantNullAnnotation=warning +org.eclipse.jdt.core.compiler.problem.redundantNullCheck=ignore +org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=ignore +org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled +org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning +org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled +org.eclipse.jdt.core.compiler.problem.suppressWarnings=enabled +org.eclipse.jdt.core.compiler.problem.syntacticNullAnalysisForFields=disabled +org.eclipse.jdt.core.compiler.problem.syntheticAccessEmulation=ignore +org.eclipse.jdt.core.compiler.problem.terminalDeprecation=warning +org.eclipse.jdt.core.compiler.problem.typeParameterHiding=warning +org.eclipse.jdt.core.compiler.problem.unavoidableGenericTypeProblems=enabled +org.eclipse.jdt.core.compiler.problem.uncheckedTypeOperation=warning +org.eclipse.jdt.core.compiler.problem.unclosedCloseable=warning +org.eclipse.jdt.core.compiler.problem.undocumentedEmptyBlock=ignore +org.eclipse.jdt.core.compiler.problem.unhandledWarningToken=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentType=warning +org.eclipse.jdt.core.compiler.problem.unlikelyCollectionMethodArgumentTypeStrict=disabled +org.eclipse.jdt.core.compiler.problem.unlikelyEqualsArgumentType=info +org.eclipse.jdt.core.compiler.problem.unnecessaryElse=ignore +org.eclipse.jdt.core.compiler.problem.unnecessaryTypeCheck=ignore +org.eclipse.jdt.core.compiler.problem.unqualifiedFieldAccess=ignore +org.eclipse.jdt.core.compiler.problem.unstableAutoModuleName=warning +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownException=ignore +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionExemptExceptionAndThrowable=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedDeclaredThrownExceptionWhenOverriding=disabled +org.eclipse.jdt.core.compiler.problem.unusedExceptionParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedImport=warning +org.eclipse.jdt.core.compiler.problem.unusedLabel=warning +org.eclipse.jdt.core.compiler.problem.unusedLocal=warning +org.eclipse.jdt.core.compiler.problem.unusedObjectAllocation=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedParameterIncludeDocCommentReference=enabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenImplementingAbstract=disabled +org.eclipse.jdt.core.compiler.problem.unusedParameterWhenOverridingConcrete=disabled +org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning +org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore +org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning +org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 47378bb04..000000000 --- a/.travis.yml +++ /dev/null @@ -1,97 +0,0 @@ -dist: bionic -language: java - -script: -- cd src && python check_names.py && cd .. - -cache: - directories: - - target/lib -after_success: - - bash <(curl -s https://codecov.io/bash) -after_failure: - - cat target/pmd.xml -branches: - only: - - master - -matrix: - include: - - name: MariaDB - jdk : oraclejdk11 - before_install: - - sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 - - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://ftp.utexas.edu/mariadb/repo/10.3/ubuntu bionic main' - - sudo apt update - - sudo apt install mariadb-server - - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - - MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test - - name : MySQL - jdk : oraclejdk11 - script: - - sudo apt-get update && sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 - - sudo apt-get remove mysql-* && wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && yes | sudo dpkg -i *.deb - - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - - MYSQL_AVAILABLE=true mvn -Dtest=TestMySQL test - - name: CockroachDB - jdk : oraclejdk11 - before_install: - - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.2.linux-amd64.tgz | tar xvz - - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach start-single-node --insecure & - - sleep 15 - - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - script: - - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - - name: TiDB - jdk : oraclejdk11 - services: - - docker - before_install: - - docker pull pingcap/tidb:latest - - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest - - sleep 15 - - sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - script: - - TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test - - name: SQLite3 - jdk : oraclejdk11 - script: - - mvn -Dtest=TestSQLite3 test - - name: DuckDB - jdk : oraclejdk11 - script: - - mvn -Dtest=TestDuckDB test - - name: DuckDB (Java 8) - jdk : openjdk8 - script: - - mvn -Dtest=TestDuckDB test - - name: DuckDB (java 13) - jdk : openjdk13 - script: - - mvn -Dtest=TestDuckDB test - - name: Misc - jdk : oraclejdk11 - script: - - mvn '-Dtest=!sqlancer.dbms.**' test - - name: PostgreSQL - jdk : oraclejdk11 - before_install: - - sudo apt-get update - - sudo apt-get --yes remove postgresql\* - - sudo apt-get install -y postgresql-12 postgresql-client-12 - - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/12/main/postgresql.conf - - sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf - - sudo service postgresql restart 12 - addons: - postgresql: "12.3" - env: - global: - - PGPORT=5432 - services: - - postgresql - before_script: - - sudo apt-get install locales - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -U postgres - - createdb test -U postgres - script: - - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..76b8833ea --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,149 @@ +# Development + +## Working with Eclipse [[Video Guide]](https://www.youtube.com/watch?v=KsuGrOLKb9Q) + +Developing SQLancer using Eclipse is expected to work well. You can import SQLancer with a single step: + +``` +File -> Import -> Existing Maven Projects -> Select the SQLancer directory as root directory -> Finish +``` +If you do not find an option to import Maven projects, you might need to install the [M2Eclipse plugin](https://www.eclipse.org/m2e/). + + +## Implementing Support for a New DBMS + +The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see enum value `NOREC` in `DuckDBOracleFactory`). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). + +Please consider the following suggestions when creating a PR to contribute a new DBMS: +* Ensure that `mvn verify -DskipTests=true` does not result in style violations. +* Add a [CI test](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) to ensure that future changes to SQLancer are unlikely to break the newly-supported DBMS. It is reasonable to do this in a follow-up PR—please indicate whether you plan to do so in the PR description. +* Add the DBMS' name to the [check_names.py](https://github.com/sqlancer/sqlancer/blob/master/src/check_names.py) script, which ensures adherence to a common prefix in the Java classes. +* Add the DBMS' name to the [README.md](https://github.com/sqlancer/sqlancer/blob/master/README.md#supported-dbms) file. +* It would be easier to review multiple smaller PRs, than one PR that contains the complete implementation. Consider contributing parts of your implementation as you work on their implementation. + +### Expected Errors + +Most statements have an [ExpectedError](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/common/query/ExpectedErrors.java) object associated with them. This object essentially contains a list of errors, one of which the database system might return if it cannot successfully execute the statement. These errors are typically added through a trial-and-error process while considering various tradeoffs. For example, consider the [DuckDBInsertGenerator](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java#L38) class, whose expected errors are specified in [DuckDBErrors](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/DuckDBErrors.java#L90). When implementing such a generator, the list of expected errors might first be empty. When running the generator for the first time, you might receive an error such as "create unique index, table contains duplicate data", indicating that creating the index failed due to duplicate data. In principle, this error could be avoided by first checking whether the column contains any duplicate values. However, checking this would be expensive and error-prone (e.g., consider string similarity, which might depend on collations); thus, the obvious choice would be to add this string to the list of expected errors, and run the generator again to check for any other expected errors. In other cases, errors might be best addressed through improvements in the generators. For example, it is typically straightforward to generate syntactically-valid statements, which is why syntax errors should not be ignored. This approach is effective in uncovering internal errors; rather than ignoring them as an expected error, report them, and see [Unfixed Bugs](#unfixed-bugs) below. + +### Bailing Out While Generating a Statement + +In some cases, it might be undesirable or even impossible to generate a specific statement type. For example, consider that SQLancer tries to execute a `DROP TABLE` statement (e.g., see [TiDBDropTableGenerator](https://github.com/sqlancer/sqlancer/blob/30948f34acc2354d6be18a70bdeeebff1e73fa48/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java)), but the database contains only a single table. Dropping the table would result in all subsequent attempts to insert data or query it to fail. Thus, in such a case, it might be more efficient to "bail out" by abandoning the current attempt to generate the statement. This can be achieved by throwing a `IgnoreMeException`. Unlike for other exceptions, SQLancer silently continues execution rather than reporting this exception to the user. + + +### Typed vs. Untyped Expression Generation + +Each DBMS implementation provides an expression generator used, for example, to generate expressions used in `WHERE` clauses. We found that DBMS can be roughly classified into "permissive" ones, which apply implicit type conversions when needed and "strict" ones, which provide only few implicit conversions and output an error when the type is unexpected. For example, consider the following test case: + +```sql +CREATE TABLE t0(c0 TEXT); +INSERT INTO t0 VALUES ('1'); +SELECT * FROM t0 WHERE c0; +``` + +If the test case is executed using MySQL, which is a permissive DBMS, the `SELECT` fetches a single row, since the content of the `c0` value is interpreted as a boolean. If the test case is executed using PostgreSQL, which is a strict DBMS, the `SELECT` is not accepted as a valid query, and PostgreSQL outputs an error `"argument of WHERE must be type boolean"`. The implementation of the expression generator depends on whether we are dealing with a permissive or a strict DBMS. Since SQLancer's main goal is to find logic bugs, we want to generate as many valid queries as possible. + +For a permissive DBMS, implementing the expression generator is easier, since the expression generator does not need to care about the type of the expression, since the DBMS will apply any necessary conversions implicitly. For MySQL, the main `generateExpression` method thus does not accept any type as an argument (see [MySQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java#L54)). This method can be called when a expression is required for, for example, a `WHERE` clause. In principle, this approach can also be used for strict DBMS, by adding errors such as `argument of WHERE must be type boolean` to the list of expected errors. However, using such an "untyped" expression generator for a strict DBMS will result in many semantically invalid queries being generated. + +For a strict DBMS, the better approach is typically to attempt to generate expressions of the expected type. For PostgreSQL, the expression generator thus expects an additional type argument (see [PostgreSQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java#L251)). This type is propagated recursively. For example, if we require a predicate for the `WHERE` clause, we pass boolean as a type. The expression generator then calls a method `generateBooleanExpression` that attempts to produce a boolean expression, by, for example, generating a comparison (e.g., `<=`). For the comparison's operands, a random type is then selected and propagated. For example, if an integer type is selected, then `generateExpression` is called with this type once for the left operand, and once for the right operand. Note that this process does not guarantee that the expression will indeed have the expected type. It might happen, for example, that the expression generator attempts to produce an integer value, but that it produces a double value instead, namely when an integer overflow occurs, which, depending on the DBMS, implicitly converts the result to a floating-point value. + +#### Supported DBMS + +Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. + +| DBMS | Status | Expression Generation | Description | +| ---------------------------- | ----------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | +| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | +| PostgreSQL | Working | Typed | | +| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | +| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | +| CockroachDB | Working | Typed | | +| TiDB | Working | Untyped | | +| DuckDB | Working | Untyped, Generic | | +| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | +| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | +| OceanBase | Working | Untyped | | +| YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | +| Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| Materialize | Working | Typed | | +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | +| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | +| DataFusion | Preliminary | Typed | Only basic SQL features are supported. | + +#### Previously Supported DBMS + +Some DBMS were once supported but subsequently removed. + +| DBMS | Pull Request | Description | +| ---------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ArangoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because ArangoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. +| CnosDB | | This implementation was removed because the CnosDB image is unstable under SQLancer's DDL load (see [cnosdb/cnosdb#2435](https://github.com/cnosdb/cnosdb/issues/2435)) and the project appears no longer maintained. | + +### Unfixed Bugs + +Often, some bugs are fixed only after an extended period, meaning that SQLancer will repeatedly report the same bug. In such cases, it might be possible to avoid generating the problematic pattern, or adding an expected error with the internal error message. Rather than, for example, commenting out the code with the bug-inducing pattern, a pattern implemented by the [TiDBBugs class](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java) should be applied. The core idea is to use a public, static flag for each issue, which is set to true as long as the issue persists (e.g., see [bug35652](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java#L55)). The work-around code is then executed—or the problematic pattern should not be generated—if the flag is set to true (e.g., [an expected error is added for bug35652](https://github.com/sqlancer/sqlancer/blob/59564d818d991d54b32fa5a79c9f733799c090f2/src/sqlancer/tidb/TiDBErrors.java#L47)). This makes it easy to later on identify and remove all such work-around code once the issue has been fixed. + +## Options + +SQLancer uses [JCommander](https://jcommander.org/) for handling options. The `MainOptions` class contains options that are expected to be supported by all DBMS-testing implementations. Furthermore, each `*Provider` class provides a method to return an additional set of supported options. + +An option can include lowercase alphanumeric characters, and hyphens. The format of the options is checked by a unit test. + +## Continuous Integration and Test Suite + +To improve and maintain SQLancer's code quality, we use multiple tools: +* The [Eclipse code formatter](https://code.revelc.net/formatter-maven-plugin/), to ensure a consistent formatting (Run `mvn formatter:format` to format all files). +* [Checkstyle](https://checkstyle.sourceforge.io/), to enforce a consistent coding standard. +* [PMD](https://pmd.github.io/), which finds programming flaws using static analysis. +* [SpotBugs](https://spotbugs.github.io/), which also uses static analysis to find bugs and programming flaws. + +You can run them using the following command: + +``` +mvn verify +``` + +We use [GitHub Actions](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) to automatically check PRs. + + +## Testing + +As part of the GitHub Actions check, we use smoke testing by running SQLancer on each supported DBMS for some minutes, to test that nothing is obviously broken. For DBMS for which all bugs have been fixed, we verify that SQLancer cannot find any further bugs (i.e., the return code is zero). + +In addition, we use [unit tests](https://github.com/sqlancer/sqlancer/tree/master/test/sqlancer) to test SQLancer's core functionality, such as random string and number generation as well as option passing. When fixing a bug, add a unit test, if it is easily possible. + +You can run the tests using the following command: + +``` +mvn test +``` + +Note that per default, the smoke testing is performed only for embedded DBMS (e.g., DuckDB and SQLite). To run smoke tests also for the other DBMS, you need to set environment variables. For example, you can run the MySQL smoke testing (and no other tests) using the following command: + +``` +MYSQL_AVAILABLE=true mvn -Dtest=TestMySQL test +``` + +For up-to-date testing commands, check out the `.github/workflows/main.yml` file. + +## Reviewing + +Reviewing is an effective way of improving code quality. Everyone is welcome to review any PRs. Currently, all PRs are reviewed at least by the main contributor, @mrigger. Contributions by @mrigger are currently not (necessarily) reviewed, which is not ideal. If you are willing to regularly and timely review PRs, indicate so in the SQLancer Slack workspace. + +## Naming Conventions + +Each class specific to a DBMS is prefixed by the DBMS name. For example, each class specific to SQLite is prefixed by `SQLite3`. The naming convention is [automatically checked](src/check_names.py). + +## Commit History + +Please pay attention to good commit messages (in particular subject lines). As basic guidelines, we recommend a blog post on [How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/) written Chris Beams, which provides 7 useful rules. Implement at least the following of those rules: +1. Capitalize the subject line. For example, write "**R**efactor the handling of indexes" rather than "**r**efactor the handling of indexes". +2. Do not end the subject line with a period. For example, write "Refactor the handling of indexes" rather than "Refactor the handling of indexes.". +3. Use the imperative mood in the subject line. For example, write "Refactor the handling of indexes" rather than "Refactoring" or "Refactor**ed** the handling of indexes". + +Please also pay attention to a clean commit history. Rather than merging with the main branch, use `git rebase` to rebase your commits on the main branch. Sometimes, it might happen that you discover an issue only after having already created a commit, for example, when an issue is found by `mvn verify` in the CI checks. Do not introduce a separate commit for such issues. If the issue was introduced by the last commit, you can fix the issue, and use `git commit --amend` to change the latest commit. If the change was introduced by one of the previous commits, you can use `git rebase -i` to change the respective commit. If you already have a number of such commits, you can use `git squash` to "collapse" multiple commits into one. For more information, you might want to read [How (and Why!) to Keep Your Git Commit History Clean](https://about.gitlab.com/blog/2018/06/07/keeping-git-commit-history-clean/) written by Kushal Pandya. + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..54f3aacaf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:21.04 + +RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install openjdk-15-jdk maven --yes --no-install-recommends + +# assumes that the project has already been built +COPY target/sqlancer-*.jar sqlancer.jar +COPY target/lib/*.jar /lib/ + +ENTRYPOINT ["java", "-jar", "sqlancer.jar"] diff --git a/README.md b/README.md index 56ec5cf43..f41e32d3c 100644 --- a/README.md +++ b/README.md @@ -1,102 +1,116 @@ -![Travis](https://travis-ci.com/sqlancer/sqlancer.svg?branch=master) -[![codecov](https://codecov.io/gh/sqlancer/sqlancer/branch/master/graph/badge.svg)](https://codecov.io/gh/sqlancer/sqlancer) -[![Twitter](https://img.shields.io/twitter/follow/sqlancer_dbms?style=social)](https://twitter.com/sqlancer_dbms) -# SQLancer +[![Build Status](https://github.com/sqlancer/sqlancer/workflows/ci/badge.svg)](https://github.com/sqlancer/sqlancer/actions) ![SQLancer](media/logo/png/sqlancer_logo_logo_pos_500.png) -SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Management Systems (DBMS) in order to find logic bugs in their implementation. We refer to logic bugs as those bugs that cause the DBMS to fetch an incorrect result set (e.g., by omitting a record). +SQLancer is a tool to automatically test Database Management Systems (DBMSs) in order to find bugs in their implementation. That is, it finds bugs in the code of the DBMS implementation, rather than in queries written by the user. SQLancer has found hundreds of bugs in mature and widely-known DBMSs. -SQLancer operates in the following two phases: +SQLancer tackles two essential challenges when automatically testing the DBMSs: +1. **Test input generation**: SQLancer implements approaches for automatically generating SQL statements. It contains various hand-written SQL generators that operate in multiple phases. First, a database schema is created, which refers to a set of tables and their columns. Then, data is inserted into these tables, along with creating various other kinds of database states such as indexes, views, or database-specific options. Finally, queries are generated, which can be validated using one of multiple result validators (also called *test oracles*) that SQLancer provides. Besides the standard approach of creating the statements in an unguided way, SQLancer also supports a test input-generation approach that is feedback-guided and aims to exercise as many unique query plans as possible based on the intuition that doing so would exercise many interesting behaviors in the database system [[ICSE '23]](https://arxiv.org/pdf/2312.17510). +2. **Test oracles**: A key innovation in SQLancer is that it provides ways to find deep kinds of bugs in DBMSs. As a main focus, it can find logic bugs, which are bugs that cause the DBMS to fetch an incorrect result set (e.g., by omitting a record). We have proposed multiple complementary test oracles such as *Ternary Logic Partitioning (TLP)* [[OOPSLA '20]](https://dl.acm.org/doi/pdf/10.1145/3428279), *Non-optimizing Reference Engine Construction (NoREC)* [[ESEC/FSE 2020]](https://arxiv.org/abs/2007.08292), *Pivoted Query Synthesis (PQS)* [[OSDI '20]](https://www.usenix.org/system/files/osdi20-rigger.pdf), *Differential Query Plans (DQP)* [[SIGMOD '24]](https://dl.acm.org/doi/pdf/10.1145/3654991), and *Constant Optimization Driven Database System Testing (CODDTest)* [SIGMOD '25]. It can also find specific categories of performance issues, which refer to cases where a DBMS could reasonably be expected to produce its result more efficiently using a technique called *Cardinality Estimation Restriction Testing (CERT)* [[ICSE '24]](https://arxiv.org/pdf/2306.00355). SQLancer can detect unexpected internal errors (e.g., an error that the database is corrupted) by declaring all potential errors that might be returned by a DBMS for a given query. Finally, SQLancer can find crash bugs, which are bugs that cause the DBMS process to terminate. For this, it uses an implicit test oracle. -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. -2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. +**Community.** We have a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. Previously, SQLancer had an account on Twitter/X [@sqlancer_dbms](https://twitter.com/sqlancer_dbms), which is no longer maintained. We have a [blog](https://sqlancer.github.io/posts/), which, as of now, contains only posts by contributors of the [Google Summer of Code project](https://summerofcode.withgoogle.com/archive/2023/organizations/sqlancer). -# Getting Started +# Getting Started [[Video Guide]](https://www.youtube.com/watch?v=lcZ6LixPH1Y) -Requirements: -* Java 8 or above -* [Maven](https://maven.apache.org/) (`sudo apt install maven` on Ubuntu) -* The DBMS that you want to test (SQLite is an embedded DBMS and is included) +Minimum Requirements: +* Java 11 or above +* [Maven](https://maven.apache.org/) -The following commands clone SQLancer, create a JAR, and start SQLancer to fuzz SQLite using Ternary Logic Query Partitioning (TLP): +The following commands clone SQLancer, create a JAR, and start SQLancer to test SQLite using [Non-optimizing Reference Engine Construction (NoREC)](https://arxiv.org/abs/2007.08292): ``` git clone https://github.com/sqlancer/sqlancer cd sqlancer mvn package -DskipTests cd target -java -jar SQLancer-0.0.1-SNAPSHOT.jar --num-threads 4 sqlite3 --oracle NoREC +java -jar sqlancer-*.jar --num-threads 4 sqlite3 --oracle NoREC ``` -If the execution prints progress information every five seconds, then the tool works as expected. Note that SQLancer might find bugs in SQLite. Before reporting these, be sure to check that they can still be reproduced when using the latest development version. The shortcut CTRL+C can be used to terminate SQLancer manually. If SQLancer does not find any bugs, it executes infinitely. The option `--num-tries` can be used to control after how many bugs SQLancer terminates. Alternatively, the option `--timeout-seconds` can be used to specify the maximum duration that SQLancer is allowed to run. +**Running and terminating.** If the execution prints progress information every five seconds, then the tool works as expected. The shortcut CTRL+C can be used to terminate SQLancer manually. If SQLancer does not find any bugs, it executes infinitely. The option `--num-tries` can be used to control after how many bugs SQLancer terminates. Alternatively, the option `--timeout-seconds` can be used to specify the maximum duration that SQLancer is allowed to run. -If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. +**Parameters.** If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of the DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. -# Potential Commercialization +**DBMSs.** To run SQLancer on SQLite, it was not necessary to install and set up a DBMS. The reason for this is that embedded DBMSs run in the same process as the application and thus require no separate installation or setup. Embedded DBMSs supported by SQLancer include DuckDB, H2, and SQLite. Their binaries are included as [JAR dependencies](https://github.com/sqlancer/sqlancer/blob/main/pom.xml). Note that any crashes in these systems will also cause a crash in the JVM on which SQLancer runs. -Due to the significant interest that we have received, we are considering to commercialize our bug-finding efforts. If you represent a company and would be interested in a bug-finding service, please approach us ([Manuel Rigger](mailto:manuel.rigger@inf.ethz.ch) and [Zhendong Su](mailto:zhendong.su@inf.ethz.ch)) with your expectations and requirements for such a service. -# Research Prototype - -This project should at this stage still be seen as a research prototype. We believe that the tool is not ready to be used. However, we have received many requests by companies, organizations, and individual developers, which is why we decided to prematurely release the tool. Expect errors, incompatibilities, lack of documentation, and insufficient code quality. That being said, we are working hard to address these issues and enhance SQLancer to become a production-quality piece of software. We welcome any issue reports, extension requests, and code contributions. - -# Testing Approaches - -| Approach | Description | -|------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Pivoted Query Synthesis (PQS) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described [here](https://arxiv.org/abs/2001.04174). PQS is the most powerful technique, but also requires more implementation effort than the other two techniques. It is currently unmaintained. | -| Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | -| Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | +# Using SQLancer -Please find the `.bib` entries [here](docs/DEVELOPMENT.md). +**Logs.** SQLancer stores logs in the `target/logs` subdirectory. By default, the option `--log-each-select` is enabled, which results in every SQL statement that is sent to the DBMS being logged. The corresponding file names are postfixed with `-cur.log`. In addition, if SQLancer detects a logic bug, it creates a file with the extension `.log`, in which the statements to reproduce the bug are logged, including only the last query that was executed along with the other statements to set up the database state. -# Supported DBMS +**Reducing bugs.** After finding a bug-inducing test input, the input typically needs to be reduced to be further analyzed, as it might contain many SQL statements that are redundant to reproduce the bug. One option is to do this manually, by removing a statement or feature at a time, replaying the bug-inducing statements, and applying the test oracle (e.g., for test oracles like TLP or NoREC, this would require checking that both queries still produce a different result). This process can be automated using a so-called [delta-debugging approach](https://www.debuggingbook.org/html/DeltaDebugger.html). SQLancer includes an experimental implementation of a delta debugging approach, which can be enabled using `--use-reducer`. In the past, we have successfully used [C-Reduce](https://embed.cs.utah.edu/creduce/), which requires specifying the test oracle in a script that can be executed by C-Reduce. -Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. +**Testing the latest DBMS version.** For most DBMSs, SQLancer supports only a previous *release* version. Thus, potential bugs that SQLancer finds could be already fixed in the latest *development* version of the DBMS. If you are not a developer of the DBMS that you are testing, we would like to encourage you to validate that the bug can still be reproduced before reporting it. We would appreciate it if you could mention SQLancer when you report bugs found by it. We would also be excited to hear about your experience using SQLancer or related use cases or extensions. -| DBMS | Status | Expression Generation | Description | -|-------------|-------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | -| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | -| PostgreSQL | Working | Typed | | -| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | -| CockroachDB | Working | Typed | | -| TiDB | Working | Untyped | | -| DuckDB | Working | Untyped, Generic | | -| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | -| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | +**Options.** SQLancer provides many options that you can use to customize its behavior. Executing `java -jar sqlancer-*.jar --help` will list them and should print output such as the following: +``` +Usage: SQLancer [options] [command] [command options] + Options: + --ast-reducer-max-steps + EXPERIMENTAL Maximum steps the AST-based reducer will do + Default: -1 + --ast-reducer-max-time + EXPERIMENTAL Maximum time duration (secs) the statement reducer will do + Default: -1 + --canonicalize-sql-strings + Should canonicalize query string (add ';' at the end + Default: true + --constant-cache-size + Specifies the size of the constant cache. This option only takes effect + when constant caching is enabled + Default: 100 +... +``` +**Which SQLancer version to use.** The recommended way to use SQLancer is to use its latest source version on GitHub. Infrequent and irregular official releases are also available on the following platforms: +* [GitHub](https://github.com/sqlancer/sqlancer/releases) +* [Maven Central](https://search.maven.org/artifact/com.sqlancer/sqlancer) +* [DockerHub](https://hub.docker.com/r/mrigger/sqlancer) -# Using SQLancer +**Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. -## Logs +**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). -SQLancer stores logs in the `target/logs` subdirectory. By default, the option `--log-each-select` is enabled, which results in every SQL statement that is sent to the DBMS being logged. The corresponding file names are postfixed with `-cur.log`. In addition, if SQLancer detects a logic bug, it creates a file with the extension `.log`, in which the statements to reproduce the bug are logged. +# Approaches and Papers -## Reducing a Bug +SQLancer has pioneered and includes multiple approaches for DBMS testing, as outlined below in chronological order. -After finding a bug, it is useful to produce a minimal test case before reporting the bug, to save the DBMS developers' time and effort. For many test cases, [C-Reduce](https://embed.cs.utah.edu/creduce/) does a great job. In addition, we have been working on a SQL-specific reducer, which we plan to release soon. +| Technique | Venue | Links | Description | +|-----------------------------------------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | +| Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | +| Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | +| Differential Query Execution (DQE) | ICSE 2023 | [Paper](https://ieeexplore.ieee.org/document/10172736) [Code](https://github.com/sqlancer/sqlancer/pull/1251) | Differential Query Execution (DQE) is a novel and general approach to detect logic bugs in SELECT, UPDATE and DELETE queries. DQE solves the test oracle problem by executing SELECT, UPDATE and DELETE queries with the same predicate φ, and observing inconsistencies among their execution results. For example, if a row that is updated by an UPDATE query with a predicate φ does not appear in the query result of a SELECT query with the same predicate φ, a logic bug is detected in the target DBMS. We append two extra columns to each table in a database to uniquely identify each row and track whether a row has been modified. We further rewrite SELECT and UPDATE queries to identify their accessed rows. DQE supports MySQL. | +| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | QPG is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | +| Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | +| Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | +| Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | -## Found Bugs +Please find the `.bib` entries [here](docs/PAPERS.md). | -We would appreciate it if you mention SQLancer when you report bugs found by it. We would also be excited to know if you are using SQLancer to find bugs, or if you have extended it to test another DBMS (also if you do not plan to contribute it to this project). SQLancer has found over 400 bugs in widely-used DBMS, which are listed [here](https://www.manuelrigger.at/dbms-bugs/). +# FAQ +**I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors?** In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). -# Community +Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html)). -We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. SQLancer's official Twitter handle is [@sqlancer_dbms](https://twitter.com/sqlancer_dbms). +**When starting SQLancer, I get an error such as "database 'test' does not exist". How can I run SQLancer without this error?** For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). +# Links -# Additional Documentation +Documentation and resources: -* [Contributing to SQLancer](docs/DEVELOPMENT.md) +* [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) +* More information on our DBMS testing efforts and the bugs we found is available [here](https://www.manuelrigger.at/dbms-bugs/). -# Additional Resources +Videos: +* [SQLancer Tutorial Playlist](https://www.youtube.com/playlist?list=PLm7ofmclym1E2LwBeSer_AAhzBSxBYDci) +* [SQLancer Talks](https://youtube.com/playlist?list=PLm7ofmclym1E9-AbYy-PkrMfHpB9VdlZJ) -* A talk on Ternary Logic Partitioning (TLP) and SQLancer is available on [YouTube](https://www.youtube.com/watch?v=Np46NQ6lqP8). -* An (older) Pivoted Query Synthesis (PQS) talk is available on [YouTube](https://www.youtube.com/watch?v=yzENTaWe7qg). -* PingCAP has implemented PQS, NoREC, and TLP in a tool called [go-sqlancer](https://github.com/chaos-mesh/go-sqlancer). -* More information on our DBMS testing efforts and the bugs we found is available [here](https://www.manuelrigger.at/dbms-bugs/). +Closely related tools: +* [go-sqlancer](https://github.com/chaos-mesh/go-sqlancer): re-implementation of some of SQLancer's approaches in Go by PingCAP +* [Jepsen](https://github.com/jepsen-io): testing of distributed (database) systems +* [SQLRight](https://github.com/PSU-Security-Universe/sqlright): coverage-guided DBMS fuzzer, also supporting NoREC and TLP +* [SQLsmith](https://github.com/anse1/sqlsmith): random SQL query generator used for fuzzing +* [Squirrel](https://github.com/s3team/Squirrel): coverage-guided DBMS fuzzer diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index ba6bb1342..000000000 --- a/codecov.yml +++ /dev/null @@ -1,16 +0,0 @@ -codecov: - require_ci_to_pass: no - notify: - after_n_builds: 10 - -coverage: - range: "50...100" - -coverage: - status: - project: - default: - threshold: 2% - patch: - default: - threshold: 2% diff --git a/configs/checkstyle.xml b/configs/checkstyle.xml index 10d21514e..530bd41d9 100644 --- a/configs/checkstyle.xml +++ b/configs/checkstyle.xml @@ -80,9 +80,13 @@ - - + + + + + @@ -169,9 +173,16 @@ - + + + + + + + + diff --git a/configs/pmd-rules.xml b/configs/pmd-rules.xml index 8098884d4..656e29f85 100644 --- a/configs/pmd-rules.xml +++ b/configs/pmd-rules.xml @@ -17,12 +17,13 @@ - + + 2 @@ -51,6 +52,7 @@ 2 + @@ -70,13 +72,13 @@ + 2 - @@ -85,5 +87,11 @@ - + + 2 + + + + + diff --git a/configs/spotbugs-exclude.xml b/configs/spotbugs-exclude.xml index 1b146bf10..7fa4de560 100644 --- a/configs/spotbugs-exclude.xml +++ b/configs/spotbugs-exclude.xml @@ -7,4 +7,19 @@ + + + + + + + + + + + + + + + diff --git a/docs/Citus README.md b/docs/Citus README.md new file mode 100644 index 000000000..7bd3ffb49 --- /dev/null +++ b/docs/Citus README.md @@ -0,0 +1,89 @@ +# SQLancer for Citus (PostgreSQL extension) + +SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Management Systems (DBMS) in order to find logic bugs in their implementation. More information about the tool can be found in the [SQLancer README](https://github.com/sqlancer/sqlancer). + +The Citus implementation of SQLancer supports the Ternary Logic Query Partitioning (TLP) test oracle. + +# Setting up + +Instructions for setting up SQLancer are described in [SQLancer - Getting Started](https://github.com/sqlancer/sqlancer#getting-started). + +Requirements for Citus: +* PostgreSQL & Citus - The steps required to build Citus from source are described in [Contributing to Citus](https://github.com/citusdata/citus/blob/master/CONTRIBUTING.md). +Optional Tools for Citus: +* [pgenv](https://github.com/thanodnl/pgenv) (for easier management of PostgreSQL versions) +* [citus_dev](https://github.com/citusdata/tools/tree/develop/citus_dev) (for easier configuration of Citus environment) + +# Using SQLancer + +The following commands run the Citus implementation of SQLancer using Ternary Logic Query Partitioning (TLP): + +``` +cd target +java -jar SQLancer-0.0.1-SNAPSHOT.jar --num-threads 4 citus --oracle QUERY_PARTITIONING +``` + +How to configure the run and how to find the output logs is explained in [SQLancer - Using SQLancer](https://github.com/sqlancer/sqlancer#using-sqlancer). + +The `--repartition` flag is a boolean optional argument specific to the Citus implementation (and therefore should be used after `citus` on the command line) that enables [repartition joins](https://docs.citusdata.com/en/v9.3/develop/api_guc.html?highlight=repartition%20join#citus-enable-repartitioned-insert-select-boolean). It is set to `true` by default. + +## Interpreting output logs + +### Current logs + +If the `--log-each-select` option is enabled, each database being tested has a corresponding `-cur.log` file that is populated with all SQL statements sent to the database. + +### Error logs + +When a bug is found in a database being tested, a corresponding `.log` file is created and is populated with all SQL statements necessary to reproduce the bug. + +1. At the top of the file is the (commented-out) error message, which provides information about the panic error/logic bug detected. +2. Below that are (commented-out) lines that give more information about the specific thread being run, including the seed value (which can be passed in as a command line flag in a later run to reproduce the same thread run). +3. Then, the steps to create the Citus database cluster are provided as commented-out lines. (Following these steps are equivalent to running `citus_dev make XXX` or following the [Citus Docs instructions](https://docs.citusdata.com/en/v9.3/installation/single_machine_debian.html) for setting up a single-machine cluster.) +4. The rest of the file (not commented-out) contains the SQL statements that prepare the testing database. +5. If the bug detected is a logic bug (the error was raised by the TLP Oracle), then the pair of buggy SELECT statements whose result sets mismatch are also appended to the end of the file as commented-out lines. + +It is important to note that these `.log` files are valid sources of SQL commands that can be passed in with the `-f` flag to the `psql` command. As long as the empty database that the file is being passed into is created with Citus support and the proper worker nodes as described in step 3, this will reproduce the state that the testing database was in when the error was detected. Then, the SQL statement(s) that caused the error can be executed to reproduce the error itself. + +Once a bug is identified, it is also possible to check whether the bug is particular to Citus or was inherited from PostgreSQL, since Citus is a PostgreSQL extension. For this, a copy of the `.log` file can be made where all Citus-specific statements (distributing a table, creating a reference table etc.) are removed. Executing this file on an empty database would produce the “vanilla” state that the database would be in without any Citus functionalities. Then, the SQL statement(s) that caused the error can be executed here to check whether the error is reproduced in “vanilla” PostgreSQL as well. + +# Maintaining & Contributing + +The instructions for setting up a development environment for contributing to SQLancer are explained in [SQLancer - Development](https://github.com/sqlancer/sqlancer/blob/master/CONTRIBUTING.md). + +## Updating expected/ignored Citus errors + +The `CitusBugs.java` file in the `src/sqlancer/citus/` directory and the `CitusCommon.java` file in the `src/sqlancer/citus/gen/` directory should be continuously updated to reflect the currently unsupported functionalities and active bugs. + +Not all SQL commands generated by SQLancer are supported by the DBMS - they might raise `SQLException`s. For instance, a command that involves an invalid casting may raise a `cannnot cast type` error. These errors do not indicate any bugs in the DBMS, which is why it is desirable to quietly ignore them if raised. The `PostgresCommon` and `CitusCommon` classes in SQLancer collect these expected errors and ensure that SQLancer does not explicitly raise an error if an expected error is thrown. + +The `addCitusErrors()` method in `CitusCommon.java` adds Citus-specific errors to the pool of expected errors. It is important to note that it is enough for a string to be a substring of the error message for an error to be ignored. This method is populated with errors that are expected in Citus behavior either because the SQL command generated by SQLancer is currently not supported by Citus, or because a bug that has already been identified has not been fixed yet and is redundantly re-appearing. Both of these, especially the latter group, are dynamic and require updating. + +The `CitusBugs` class in `CitusBugs.java` is an interface between [issues](https://github.com/citusdata/citus/issues?q=is%3Aissue+label%3Asqlancer) opened in the Citus GitHub repository and the bugs listed in the `addCitusErrors()` method in `CitusCommon.java`. Each bug is assigned a corresponding boolean variable, which can be switched to `false` (uninitialized) when the error is fixed on the Citus master branch. + +### What to do: new bug found + +If the bug found is a panic error, i.e. NOT a logic bug (mismatch in result sets identified by the TLP Oracle), this error should be added to the `CitusBugs` class and the `addCitusErrors()` method. +1. Open an issue for the bug in the [Citus GitHub repository](https://github.com/citusdata/citus/issues?q=is%3Aissue+label%3Asqlancer+), and tag the issue with the `sqlancer` label. +2. Add a boolean variable associated with this issue to the `CitusBugs` class and set it to `true`. +3. Add the error message to the `addCitusErrors()` method wrapped inside an if-statement referring to the boolean created in the `CitusBugs` class. + +If the bug found is a logic bug, i.e. a mismatch in result sets identified by the TLP Oracle, perform step 1 only. + +### What to do: bug fixed + +If the bug fixed was a panic error, i.e. NOT a logic bug (mismatch in result sets identified by the TLP Oracle), the boolean in the `CitusBugs` class corresponding to the issue resolved should be set to `false` (uninitialized) once the fix is merged to the Citus master branch. + +If the bug found was a logic bug, i.e. a mismatch in result sets identified by the TLP Oracle, no actions are necessary. + +### What to do: change in Citus support for PostgreSQL commands + +An error that was previously raised by Citus due to unsupported PostgreSQL functionalities can be removed from the `addCitusErrors()` method if Citus begins supporting this functionality. + +## Modifying the database environment setup + +The `CitusProvider.java` file in the `src/sqlancer/citus/` directory includes the methods for connecting to an existing database and creating the distributed database environment, as well as for preparing the environment for testing (creation of local, distributed, and reference tables and modification of these tables). + +## Modifying JOINs in the SELECT statements generated for testing + +The `CitusTLPBase.java` file in the `src/sqlancer/citus/oracle/tlp/` directory includes the methods for generating JOIN clauses, which can be modified to alter the scope of the JOINs. \ No newline at end of file diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md deleted file mode 100644 index c54e44b4f..000000000 --- a/docs/DEVELOPMENT.md +++ /dev/null @@ -1,46 +0,0 @@ -# Development - -## Working with Eclipse - -Developing SQLancer using Eclipse is expected to work well. You can import SQLancer with a single step: - -``` -File -> Import -> Existing Maven Projects -> Select the SQLancer directory as root directory -> Finish -``` -If you do not find an option to import Maven projects, you might need to install the [M2Eclipse plugin](https://www.eclipse.org/m2e/). - - -## Implementing Support for a New DBMS - -The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see `DuckDBNoRECOracle` or `DuckDBQueryPartitioningWhereTester` for TLP). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). - -## Options - -SQLancer uses [JCommander](https://jcommander.org/) for handling options. The `MainOptions` class contains options that are expected to be supported by all DBMS-testing implementations. Furthermore, each `*Provider` class provides a method to return an additional set of supported options. - -An option can include lowercase alphanumeric characters, and hyphens. The format of the options is checked by a unit test. - -## Continuous Integration and Test Suite - -To improve and maintain SQLancer's code quality, we use multiple tools: -* The [Eclipse code formatter](https://code.revelc.net/formatter-maven-plugin/), to ensure a consistent formatting (Run `mvn formatter:format` to format all files). -* [Checkstyle](https://checkstyle.sourceforge.io/), to enforce a consistent coding standard. -* [PMD](https://pmd.github.io/), which finds programming flaws using static analysis. -* [SpotBugs](https://spotbugs.github.io/), which also uses static analysis to find bugs and programming flaws. - -You can run them using the following command: - -``` -mvn verify -``` - -We use [Travis-CI](https://travis-ci.com/) to automatically check PRs. - - -## Testing - -We found that bugs in SQLancer are quickly found and easy to debug when testing the DBMS. However, it would still be preferable to automatically check that SQLancer still executes as expected. To this end, we would like to add smoke testing for each DBMS to test that the respective testing implementation is not obviously broken, see [here](https://github.com/sqlancer/sqlancer/issues/3). - -## Naming Conventions - -Each class specific to a DBMS is prefixed by the DBMS name. For example, each class specific to SQLite is prefixed by `SQLite3`. The naming convention is [automatically checked](src/check_names.py). diff --git a/docs/PAPERS.md b/docs/PAPERS.md index 0ed6390c7..a42b42c12 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -1,6 +1,6 @@ # Papers -The testing approaches implemented in SQLancer are described in the three papers below. +The testing approaches implemented in SQLancer are described in the four papers below. ## Testing Database Engines via Pivoted Query Synthesis @@ -8,16 +8,19 @@ This paper describes PQS, a testing approach to detect various kinds of logic bu ``` @inproceedings{Rigger2020PQS, - author={Manuel Rigger and Zhendong Su}, - title={Testing Database Engines via Pivoted Query Synthesis}, - year={2020}, - url={https://arxiv.org/abs/2001.04174} + title = {Testing Database Engines via Pivoted Query Synthesis}, + booktitle = {14th {USENIX} Symposium on Operating Systems Design and Implementation ({OSDI} 20)}, + year = {2020}, + address = {Banff, Alberta}, + url = {https://www.usenix.org/conference/osdi20/presentation/rigger}, + publisher = {{USENIX} Association}, + month = nov, } ``` ## Detecting Optimization Bugs in Database Engines via Non-Optimizing Reference Engine Construction -This paper describes NoREC, a metamorphic testing approach to detect optimization bugs, that is, logic bugs that affect the query optimizer. A preprint is available [here](https://www.manuelrigger.at/preprints/NoREC.pdf). +This paper describes NoREC, a metamorphic testing approach to detect optimization bugs, that is, logic bugs that affect the query optimizer. A preprint is available [here](https://arxiv.org/abs/2007.08292). ``` @inproceedings{Rigger2020NoREC, @@ -26,18 +29,77 @@ This paper describes NoREC, a metamorphic testing approach to detect optimizatio booktitle = {Proceedings of the 2020 28th ACM Joint Meeting on European Software Engineering Conference and Symposium on the Foundations of Software Engineering}, series={ESEC/FSE 2020}, location={Sacramento, California, United States}, - year={2020} + year={2020}, + doi={10.1145/3368089.3409710} } ``` ## Ternary Logic Partitioning: Detecting Logic Bugs in Database Management Systems -This paper describes TLP, a metamorphic testing approach that can detect various kinds of logic bugs and is applicable also test features such as aggregate functions. A preprint is available [here](https://www.manuelrigger.at/preprints/TLP.pdf). +This paper describes TLP, a metamorphic testing approach that can detect various kinds of logic bugs and is applicable also to test features such as aggregate functions. A preprint is available [here](https://www.manuelrigger.at/preprints/TLP.pdf). ``` -@inproceedings{Rigger2020TLP, +@article{Rigger2020TLP, author={Manuel Rigger and Zhendong Su}, - title={Ternary Logic Partitioning: Detecting Logic Bugs in Database Management Systems}, - year={2020} + title={Finding Bugs in Database Systems via Query Partitioning}, + journal = {Proc. ACM Program. Lang.}, + number = {OOPSLA}, + year={2020}, + doi={10.1145/3428279}, + volume={4}, + articleno={211} +} +``` + +## Testing Database Engines via Query Plan Guidance + +This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. A preprint is available [here](http://bajinsheng.github.io/assets/pdf/qpg_icse23.pdf). + +``` +@inproceedings{Ba2023QPG, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {Testing Database Engines via Query Plan Guidance}, + booktitle = {The 45th International Conference on Software Engineering (ICSE'23)}, + year = {2023}, + month = may +} +``` + +## CERT: Finding Performance Issues in Database Systems Through the Lens of Cardinality Estimation + +This paper describes CERT, a testing approach to find performance issues by inspecting inconsistent estimated cardinalities. A preprint is available [here](https://bajinsheng.github.io/assets/pdf/cert_icse24.pdf). + +``` +@inproceedings{cert, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {CERT: Finding Performance Issues in Database Systems Through the Lens of Cardinality Estimation}, + booktitle = {The 46th International Conference on Software Engineering (ICSE'24)}, + year = {2024}, + month = apr, +} +``` + +## Keep It Simple: Testing Databases via Differential Query Plans + +This paper describes DQP, a testing approach to find logic bugs in database systems by comparing the query plans of different database systems. A preprint is available [here](https://bajinsheng.github.io/assets/pdf/dqp_sigmod24.pdf). + +``` +@article{dqp, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {Keep It Simple: Testing Databases via Differential Query Plans}, + year = {2024}, + issue_date = {June 2024}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + journal = {Proceeding of ACM Management of Data (SIGMOD'24)}, + month = jun } ``` + +# Comparing SQLancer With Other Tools that Find Logic Bugs + +If you want to fairly compare other tools with SQLancer, we would be glad to provide feedback (e.g., feel free to send an email to manuel.rigger@inf.ethz.ch). We have the following general recommendations and comments: +* PostgreSQL and SQLite are DBMSs that we comprehensively tested, and where all or most of the bugs that SQLancer could find were fixed. We believe these two systems to be the most challenging test targets. Finding bugs that the approaches implemented in SQLancer overlooked in these systems might thus best demonstrate a new approach's effectiveness. For some other DBMSs like MySQL and MariaDB, SQLancer could still detect unreported bugs; we stopped testing these DBMSs and reporting bugs due to the large number of unfixed bugs. +* We programmatically disabled the generation of features that are likely to trigger known bugs (e.g., see [TiDB](https://github.com/sqlancer/sqlancer/blob/master/src/sqlancer/tidb/TiDBBugs.java)). If a comparison investigates metrics such as code coverage that is achieved when fuzzing a DBMS, it might be desirable to enable the generation of such features. +* For the default SQLite JDBC driver, a number of extensions (e.g., the [soundex function](https://sqlite.org/lang_corefunc.html#soundex)) are disabled by default, which is why they are also disabled by default in the DBMS' options (e.g., see [SQLite3Options](https://github.com/sqlancer/sqlancer/blob/c71b9741f680f4877fc5047445787ed184a5a5e0/src/sqlancer/sqlite3/SQLite3Options.java#L67)). To investigate metrics such as code coverage, it might again be desirable to enable such options. +* The maximum expression depth (see the `--max-expression-depth` option), the number of queries issued per database (see the `--num-queries` option), and the number of tables and views that are created (currently, SQLancer does not have an option to set these) significantly influence the tool's effectiveness and performance characteristics. It might be desirable to experiment with different values for the expression depth (e.g., values between 2 and 4), the number of queries (1000-100,000), as well as the number of tables and views. diff --git a/docs/QueryPlanGuidance.md b/docs/QueryPlanGuidance.md new file mode 100644 index 000000000..bb467461b --- /dev/null +++ b/docs/QueryPlanGuidance.md @@ -0,0 +1,66 @@ +# Query Plan Guidance +Query Plan Guidance (QPG) is a test case generation method that attempts to explore unseen query plans. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. Here, we document all mutators in which we choose the most promising one that may help covering more unique query plans to execute. + +# Mutators +All mutators are listed below and implemented in the enumeration variables `Action` in the `XXDBProvider.java` file of each DBMS. +The `Mutator` column includes the items in the `Action` enumeration variable. +The `Example` column includes an example of a realistic statement generated by this mutator. +The `Description` column includes an explanation of what the mutator does. +The `More unique query plans...` column explains why applying this mutator may help covering more unique query plans. + + +|DBMS |Mutator |Example |Description |More unique query plans may be covered because of | +|-----------|---------------------|--------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|--------------------------------------------------------| +|SQLite |PRAGMA |PRAGMA automatic_index true; |It modifies database options. |different options that decide how to execute statements.| +|SQLite |CREATE_INDEX |CREATE INDEX i0 ON t0 WHERE c0 ISNULL; |It adds a new index on a table. |subsequent differnt logic of querying data. | +|SQLite |CREATE_VIEW |CREATE VIEW v0(c0) AS SELECT DISTINCT ABS(t0.c2) FROM t0; |It adds a new view from existing tables. |more possible execution logics on the view. | +|SQLite |CREATE_TABLE |CREATE TABLE t0 (c0 INT CHECK ((c0) BETWEEN (1) AND (10)) ); |It adds a new table. |more possible execution logics on the table. | +|SQLite |CREATE_VIRTUALTABLE |CREATE VIRTUAL TABLE vt1 USING fts5(c0 UNINDEXED); |It adds a new table with fts5 feature. |more possible execution logics on the table with fts5. | +|SQLite |CREATE_RTREETABLE |CREATE VIRTUAL TABLE rt0 USING rtree_i32(c0, c1, c2, c3, c4); |It adds a new table with rtree feature. |more possible execution logics on the table with rtree. | +|SQLite |INSERT |INSERT INTO t0(c0, c1) VALUES ('lrd+a*', NULL); |It inserts a new row to a table. |subsequent different logic of querying data. | +|SQLite |DELETE |DELETE FROM t0 WHERE (c0>3); |It deletes specific rows from a table. |subsequent different logic of querying data. | +|SQLite |ALTER |ALTER TABLE t0 ADD COLUMN c39 REAL; |It changes the schema of a table. |more possible execution logics on the changed table. | +|SQLite |UPDATE |UPDATE t0 SET (c2, c0)=(-944, 'L((xA') WHERE t0.c1; |It updates specific data of a table. |subsequent different logic of querying data. | +|SQLite |DROP_INDEX |DROP INDEX i0; |It drops an index. |subsequent different logic of querying data. | +|SQLite |DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|SQLite |DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|SQLite |VACUUM |VACUUM main; |It rebuilds the database file. |subsequent different logic of querying data. | +|SQLite |REINDEX |REINDEX t0; |It drops and recreates indexes from scratch. |subsequent different logic of querying data. | +|SQLite |ANALYZE |ANALYZE t0; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|SQLite |EXPLAIN |EXPLAIN SELECT * FROM t0; |It obtains query plan of a query. |subsequent different logic of querying data. | +|SQLite |CHECK_RTREE_TABLE |SELECT rtreecheck('rt0'); |It runs an integrity check on a table. |subsequent different logic of querying data. | +|SQLite |VIRTUAL_TABLE_ACTION |INSERT INTO vt0(vt0) VALUES('rebuild'); |It changes the options of a virtual table. |subsequent different logic of querying data. | +|SQLite |MANIPULATE_STAT_TABLE|INSERT INTO sqlite_stat1 VALUES('rt0', 't1', '2'); |It changes the table that stores statistics of all tables. |subsequent different logic of querying data. | +|SQLite |TRANSACTION_START |BEGIN TRANSACTION; |All statements after this will not be committed. |subsequent different logic of querying data. | +|SQLite |ROLLBACK_TRANSACTION |ROLLBACK TRANSACTION; |All statements after last BEGIN are dropped. |subsequent different logic of querying data. | +|SQLite |COMMIT |COMMIT; |All statements after last BEGIN are committed |subsequent different logic of querying data. | +|TiDB |CREATE_TABLE |CREATE TABLE t1(c0 INT); |It adds a new table. |more possible execution logics on the table. | +|TiDB |CREATE_INDEX |CREATE INDEX i0 ON t0(c0(250) ASC) KEY_BLOCK_SIZE 1564693810209727437; |It adds a new index on a table. |subsequent differnt logic of querying data. | +|TiDB |VIEW_GENERATOR |CREATE VIEW v0(c0, c1) AS SELECT t1.c0, ((t1.c0)REGEXP('8')) FROM t1; |It adds a new view from existing tables. |more possible execution logics on the view. | +|TiDB |INSERT |INSERT INTO t0(c0) VALUES (-16387); |It inserts a new row to a table. |subsequent different logic of querying data. | +|TiDB |ALTER_TABLE |ALTER TABLE t1 ADD PRIMARY KEY(c0); |It changes the schema of a table. |more possible execution logics on the changed table. | +|TiDB |TRUNCATE |TRUNCATE t0; |It drops all rows of a table. |subsequent different logic of querying data. | +|TiDB |UPDATE |UPDATE t0 SET c0='S' WHERE t0.c0; |It updates specific data of a table. |subsequent different logic of querying data. | +|TiDB |DELETE |DELETE FROM t0 ORDER BY CAST(t0.c0 AS CHAR) DESC; |It deletes specific rows from a table. |subsequent different logic of querying data. | +|TiDB |SET |set @@tidb_max_chunk_size=8864; |It modifies database options. |different options that decide how to execute statements.| +|TiDB |ADMIN_CHECKSUM_TABLE |ADMIN CHECKSUM TABLE t0; |it calculate the checksum for a table. |subsequent different logic of querying data. | +|TiDB |ANALYZE_TABLE |ANALYZE TABLE t1 WITH 174 BUCKETS; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|TiDB |DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|TiDB |DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|CockroachDB|CREATE_TABLE |CREATE TABLE t1 (c0 INT4, c1 VARBIT(44) UNIQUE DEFAULT (B'000'), CONSTRAINT "primary" PRIMARY KEY(c1 ASC, c0 ASC));|It adds a new table. |more possible execution logics on the table. | +|CockroachDB|CREATE_INDEX |CREATE INDEX ON t0(rowid); |It adds a new index on a table. |subsequent differnt logic of querying data. | +|CockroachDB|CREATE_VIEW |CREATE VIEW v0(c0) AS SELECT DISTINCT MIN(TIMETZ '1970-01-11T12:19:44') FROM t0; |It adds a new view from existing tables. |more possible execution logics on the view. | +|CockroachDB|CREATE_STATISTICS |CREATE STATISTICS s0 FROM t2; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|CockroachDB|INSERT |INSERT INTO t1 (rowid, c0) VALUES(NULL, true) ON CONFLICT (c0) DO NOTHING ; |It inserts a new row to a table. |subsequent different logic of querying data. | +|CockroachDB|UPDATE |UPDATE t0@{FORCE_INDEX=t0_pkey} SET c0=t0.c0; |It updates specific data of a table. |subsequent different logic of querying data. | +|CockroachDB|SET_SESSION |SET SESSION BYTEA_OUTPUT=escape; |It changes session configurations. |different options that decide how to execute statements.| +|CockroachDB|SET_CLUSTER_SETTING |SET CLUSTER SETTING sql.query_cache.enabled=true; |It changes cluster configurations. |different options that decide how to execute statements.| +|CockroachDB|DELETE |DELETE from t0; |It deletes specific rows from a table. |subsequent different logic of querying data. | +|CockroachDB|TRUNCATE |TRUNCATE TABLE t1 CASCADE; |It drops all rows of a table. |subsequent different logic of querying data. | +|CockroachDB|DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|CockroachDB|DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|CockroachDB|COMMENT_ON |COMMENT ON INDEX t0_c0_key IS '|?'; |It changes schema of a table. |subsequent different logic of querying data. | +|CockroachDB|SHOW |SHOW LOCALITY; |It lists detailed information of active queries. |subsequent different logic of querying data. | +|CockroachDB|EXPLAIN |EXPLAIN SELECT * FROM t0; |It obtains query plan of a query. |subsequent different logic of querying data. | +|CockroachDB|SCRUB |EXPERIMENTAL SCRUB table t0; |It checks data corruption of a table. |subsequent different logic of querying data. | +|CockroachDB|SPLIT |ALTER INDEX t0@t0_c0_key SPLIT AT VALUES (NULL); |It changes the indexes. |subsequent different logic of querying data. | diff --git a/docs/testCaseReduction.md b/docs/testCaseReduction.md new file mode 100644 index 000000000..ee317f791 --- /dev/null +++ b/docs/testCaseReduction.md @@ -0,0 +1,50 @@ +# Test Case Reduction +SQLancer generates a large number of statements, but not all of them are relevant to the bug. To automatically reduce the test cases, two reducers were implemented: the statement reducer and the AST-based reducer. + +## Statement Reducer +The statement reducer utilizes the delta-debugging technique to remove irrelevant statements. More details of delta-debugging could be found in this paper: [Simplifying and Isolating Failure-Inducing Input](https://www.cs.purdue.edu/homes/xyzhang/fall07/Papers/delta-debugging.pdf). + +Using the statement reducer, SQLancer reduces the set of statements to a minimal subset that reproduces the bug. + +## AST-Based Reducer +The AST-based reducer can shorten a statement by applying AST level transformations, including removing unnecessary clauses, irrelevant elements in a list, simplify complicated expressions and etc. + +The transformations are implemented by [JSQLParser](https://github.com/JSQLParser/JSqlParser), a RDBMS agnostic SQL statement parser that can translate SQL statements into a traversable hierarchy of Java classes. JSQLParser provides support for the SQL standard as well as major SQL dialects. The AST-based reducer works for any SQL dialects that can be parsed by this tool. + +## Implementing reproducer +Determining whether a bug persists after reducing statements +is an undecidable task for general transformations. +In practice, reducers use the [reproducer](../src/sqlancer/Reproducer.java) to determine +if a bug remains after statements have been removed or modified. +The reducer's responsibility is to verify if the current state, +formed by the pared-down statements, +continues to yield incorrect results for specific queries. + +Different oracles have distinct logic for determination, +meaning a universal reproducer doesn't exist. +Each oracle type needs its own reproducer implementation. +If reproducer is not implemented for specific oracle, +test case reduction is not available while using the oracle. + +Oracles for which reproducers have currently been implemented include: +1. for [`SQLite3NoRECOracle`](../src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java) +2. for [`TiDBTLPWhereOracle`](../src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java) + +## Using reducers +Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-ast` parameter so that the AST-based reduction is applied. + +Note: if `--reduce-ast` is set, `--use-reducer` option must be enabled first. + +There are also options to define timeout seconds and max steps of reduction for both statement reducer and AST-based reducer. + +``` +--statement-reducer-max-steps= +--statement-reducer-max-time= +--ast-reducer-max-steps= +--ast-reducer-max-time= +``` + +## Reduction logs +If test-case reduction is enabled, each time the reducer performs a reduction step successfully,it prints the reduced statements to the log file, overwriting the previous ones. + +The log files will be stored in the following format: `logs//reduce/-reduce.log`. For instance, if the tested DBMS is SQLite3 and the current database is named database0, the log file will be located at `logs/sqlite3/reduce/database0-reduce.log`. diff --git a/pom.xml b/pom.xml index 8669c0664..c4bc71f82 100644 --- a/pom.xml +++ b/pom.xml @@ -2,9 +2,38 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - SQLancer - SQLancer - 0.0.1-SNAPSHOT + com.sqlancer + sqlancer + 2.0.0 + SQLancer + http://www.sqlancer.com/ + SQLancer finds logic bugs in Database Management Systems through automatic testing + + + MIT License + https://github.com/sqlancer/sqlancer/blob/master/LICENSE.md + repo + + + + + mrigger + Manuel Rigger + manuel.rigger@inf.ethz.ch + ETH Zurich + https://ethz.ch/ + https://www.manuelrigger.at/ + + architect + developer + + + + + https://github.com/sqlancer/sqlancer/ + scm:git:git://github.com/sqlancer/sqlancer.git + scm:git:ssh://github.com:sqlancer/sqlancer.git + UTF-8 @@ -12,6 +41,46 @@ src test + + org.apache.maven.plugins + maven-shade-plugin + 3.4.0 + + + package + + shade + + + + + com.beust:jcommander + + + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + package + + run + + + + + + + + + + + org.apache.maven.plugins maven-surefire-plugin @@ -20,7 +89,7 @@ org.jacoco jacoco-maven-plugin - 0.8.5 + 0.8.12 @@ -37,11 +106,15 @@ + org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.10.1 - 8 - 8 + 11 + 11 + + ${project.basedir}/.settings/org.eclipse.jdt.core.prefs + eclipse true true @@ -50,19 +123,24 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.8.6 + 2.13.0 org.eclipse.jdt ecj - 3.22.0 + 3.28.0 + + + org.codehaus.plexus + plexus-compiler-api + 2.13.0 net.revelc.code.formatter formatter-maven-plugin - 2.12.0 + 2.20.0 eclipseformat @@ -76,6 +154,7 @@ org.apache.maven.plugins maven-dependency-plugin + 3.4.0 copy-dependencies @@ -88,6 +167,7 @@ false false true + jcommander @@ -95,8 +175,9 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.0 + 3.3.0 + true true @@ -123,7 +204,14 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.1.1 + 3.2.0 + + + com.puppycrawl.tools + checkstyle + 10.5.0 + + configs/checkstyle.xml @@ -143,7 +231,7 @@ org.apache.maven.plugins maven-pmd-plugin - 3.13.0 + 3.14.0 pmd @@ -164,7 +252,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.0.4 + 4.7.3.0 spotbugs @@ -183,51 +271,165 @@ + + com.google.auto.service + auto-service + 1.0.1 + com.beust jcommander - 1.78 + 1.82 org.postgresql postgresql - 42.2.14 + 42.5.1 + + + com.ing.data + cassandra-jdbc-wrapper + 4.7.0 + + + com.yugabyte + jdbc-yugabytedb + 42.3.5-yb-1 org.xerial sqlite-jdbc - 3.32.3 + 3.49.1.0 - mysql - mysql-connector-java - 8.0.20 + com.mysql + mysql-connector-j + 9.7.0 org.mariadb.jdbc mariadb-java-client - 2.6.1 + 3.1.0 org.duckdb duckdb_jdbc - 0.1.9 + 1.3.0.0 + + + com.facebook.presto + presto-jdbc + 0.283 org.junit.jupiter junit-jupiter-engine - 5.6.2 + 5.11.2 test org.slf4j - slf4j-simple - 1.7.30 + slf4j-simple + 2.0.6 ru.yandex.clickhouse clickhouse-jdbc - 0.2.4 + 0.3.2 + + + com.h2database + h2 + 2.3.232 + + + org.mongodb + mongodb-driver-sync + 4.1.1 + + + com.arangodb + arangodb-java-driver + 6.9.0 + + + org.questdb + questdb + 6.5.3 + + + org.hsqldb + hsqldb + 2.7.4 + runtime + + + org.apache.commons + commons-csv + 1.9.0 + + + com.github.jsqlparser + jsqlparser + 4.6 + + + org.apache.arrow + flight-sql-jdbc-driver + 16.1.0 + + + org.apache.hive + hive-jdbc + 3.1.2 + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + org.apache.hive + hive-serde + 4.0.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + org.apache.hive + hive-cli + 4.0.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + org.apache.logging.log4j + log4j-api + 2.24.3 + + + org.apache.logging.log4j + log4j-core + 2.24.3 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.3 + + + org.apache.hadoop + hadoop-common + 3.2.4 @@ -235,8 +437,115 @@ org.apache.maven.plugins maven-jxr-plugin - 3.0.0 + 3.3.0 + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + jdk-8-config + + [1.3,1.9) + + + ${java.home}/../bin/javadoc + + + + jdk-11-config + + [11,) + + + ${java.home}/bin/javadoc + + + + release-steps + + + releaseBuild + true + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.4.1 + + 8 + + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + + + + datafusion-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.3.0 + + + **/TestDataFusion.java + + --add-opens java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + + diff --git a/src/check_names.py b/src/check_names.py index f76b881ab..453580f88 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -1,28 +1,55 @@ import os +import sys +from typing import List -def get_java_files(directory): - java_files = [] - for root, dirs, files in os.walk(directory): - for f in files: - if f.endswith('.java'): - java_files.append(f) - return java_files - -def verify_prefix(prefix, files): - if len(files) == 0: - print(prefix + ' directory does not contain any files!') - exit(-1) - for f in files: - if not f.startswith(prefix): - print('The class name of ' + f + ' does not start with ' + prefix) - exit(-1) - -# TODO: ClickHouse (wait for https://github.com/sqlancer/sqlancer/pull/39) -verify_prefix('CockroachDB', get_java_files("sqlancer/cockroachdb/")) -verify_prefix('DuckDB', get_java_files("sqlancer/duckdb")) -verify_prefix('MariaDB', get_java_files("sqlancer/mariadb/")) -verify_prefix('MySQL', get_java_files("sqlancer/mysql/")) -verify_prefix('Postgres', get_java_files("sqlancer/postgres/")) -verify_prefix('SQLite3', get_java_files("sqlancer/sqlite3/")) -verify_prefix('TiDB', get_java_files("sqlancer/tidb/")) +def get_java_files(directory_path: str) -> List[str]: + java_files: List[str] = [] + for root, dirs, files in os.walk(directory_path): + for f in files: + if f.endswith('.java'): + java_files.append(f) + return java_files + + +def verify_one_db(prefix: str, files: List[str]): + print('checking database, name: {0}, files: {1}'.format(prefix, files)) + if len(files) == 0: + print(prefix + ' directory does not contain any files!', file=sys.stderr) + exit(-1) + for f in files: + if not f.startswith(prefix): + print('The class name of ' + f + ' does not start with ' + prefix, file=sys.stderr) + exit(-1) + print('checking database pass: ', prefix) + + +def verify_all_dbs(name_to_files: dict[str:List[str]]): + for db_name, files in name_to_files.items(): + verify_one_db(db_name, files) + + +if __name__ == '__main__': + cwd = os.getcwd() + print("Current working directory: {0}".format(cwd)) + name_to_files: dict[str:List[str]] = dict() + name_to_files["Citus"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "citus")) + name_to_files["ClickHouse"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "clickhouse")) + name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) + name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) + name_to_files["DataFusion"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "datafusion")) + name_to_files["DuckDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "duckdb")) + name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) + name_to_files["HSQLDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "hsqldb")) + name_to_files["MariaDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mariadb")) + name_to_files["Materialize"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "materialize")) + name_to_files["MySQL"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mysql")) + name_to_files["OceanBase"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "oceanbase")) + name_to_files["Postgres"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "postgres")) + name_to_files["Presto"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "presto")) + name_to_files["QuestDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "questdb")) + name_to_files["SQLite3"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "sqlite3")) + name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) + name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes + name_to_files["Doris"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "doris")) + verify_all_dbs(name_to_files) diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java new file mode 100644 index 000000000..f9468af76 --- /dev/null +++ b/src/sqlancer/ASTBasedReducer.java @@ -0,0 +1,144 @@ +package sqlancer; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.transformations.RemoveClausesOfSelect; +import sqlancer.transformations.RemoveColumnsOfSelect; +import sqlancer.transformations.RemoveElementsOfExpressionList; +import sqlancer.transformations.RemoveRowsOfInsert; +import sqlancer.transformations.RemoveUnions; +import sqlancer.transformations.RoundDoubleConstant; +import sqlancer.transformations.SimplifyConstant; +import sqlancer.transformations.SimplifyExpressions; +import sqlancer.transformations.Transformation; + +public class ASTBasedReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> + implements Reducer { + + private final DatabaseProvider provider; + + @SuppressWarnings("unused") + private G state; + private G newGlobalState; + private Reproducer reproducer; + + private List> reducedStatements; + // statement after reduction. + + public ASTBasedReducer(DatabaseProvider provider) { + this.provider = provider; + } + + @SuppressWarnings("unchecked") + private void updateStatements(String queryString, int index) { + boolean couldAffectSchema = queryString.contains("CREATE TABLE") || queryString.contains("EXPLAIN"); + reducedStatements.set(index, (Query) new SQLQueryAdapter(queryString, couldAffectSchema)); + } + + @SuppressWarnings("unchecked") + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + this.state = state; + this.newGlobalState = newGlobalState; + this.reproducer = reproducer; + + long maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + long maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); + + List> initialBugInducingStatements = state.getState().getStatements(); + newGlobalState.getState().setStatements(new ArrayList<>(initialBugInducingStatements)); + + List transformations = new ArrayList<>(); + + transformations.add(new RemoveUnions()); + transformations.add(new RemoveClausesOfSelect()); + transformations.add(new RemoveRowsOfInsert()); + transformations.add(new RemoveColumnsOfSelect()); + transformations.add(new RemoveElementsOfExpressionList()); + transformations.add(new SimplifyExpressions()); + transformations.add(new SimplifyConstant()); + transformations.add(new RoundDoubleConstant()); + + Transformation.setBugJudgement(() -> { + try { + return this.bugStillTriggers(); + } catch (Exception ignored) { + } + return false; + }); + + boolean observeChange; + reducedStatements = new ArrayList<>(); + for (Query query : initialBugInducingStatements) { + reducedStatements.add((Query) query); + } + + Instant startTime = Instant.now(); + reduceProcess: do { + observeChange = false; + for (Transformation t : transformations) { + for (int i = 0; i < reducedStatements.size(); i++) { + + Instant currentTime = Instant.now(); + if (maxReduceTime != MainOptions.NO_REDUCE_LIMIT + && Duration.between(startTime, currentTime).getSeconds() >= maxReduceTime) { + break reduceProcess; + } + + if (maxReduceSteps != MainOptions.NO_REDUCE_LIMIT + && Transformation.getReduceSteps() >= maxReduceSteps) { + break reduceProcess; + } + + Query query = reducedStatements.get(i); + boolean initFlag = t.init(query.getQueryString()); + int index = i; + t.setStatementChangedCallBack((statementString) -> { + updateStatements(statementString, index); + }); + + if (!initFlag) { + System.out.println("Error when parsing the statement at transformer :" + t); + continue; + } + t.apply(); + observeChange |= t.changed(); + } + } + } while (observeChange); + + newGlobalState.getState().setStatements(new ArrayList<>(reducedStatements)); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "AST-based reduction finished; the following statements remain"); + } + + public boolean bugStillTriggers() throws Exception { + try (C con2 = provider.createDatabase(newGlobalState)) { + newGlobalState.setConnection(con2); + List> candidateStatements = new ArrayList<>(reducedStatements); + newGlobalState.getState().setStatements(new ArrayList<>(candidateStatements)); + + for (Query s : candidateStatements) { + try { + s.execute(newGlobalState); + } catch (Throwable ignoredException) { + // ignore + } + } + try { + if (reproducer.bugStillTriggers(newGlobalState)) { + newGlobalState.getLogger().logReduced(newGlobalState.getState()); + return true; + } + } catch (Throwable ignoredException) { + + } + } + return false; + } +} diff --git a/src/sqlancer/AbstractAction.java b/src/sqlancer/AbstractAction.java index d2181e041..db218ebe8 100644 --- a/src/sqlancer/AbstractAction.java +++ b/src/sqlancer/AbstractAction.java @@ -1,9 +1,19 @@ package sqlancer; -import java.sql.SQLException; +import sqlancer.common.query.Query; public interface AbstractAction { - Query getQuery(G globalState) throws SQLException; + Query getQuery(G globalState) throws Exception; + + /** + * Specifies whether it makes sense to request a {@link Query}, when the previous call to {@link #getQuery(Object)} + * returned a query that failed executing. + * + * @return whether retrying getting queries makes sense, if the first query failed executing. + */ + default boolean canBeRetried() { + return true; + } } diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 45d3b5d74..cee290924 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -1,14 +1,18 @@ package sqlancer; import java.io.IOException; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + public final class ComparatorHelper { private ComparatorHelper() { @@ -29,11 +33,11 @@ static boolean equals(double a, double b) { return true; } // If the difference is less than epsilon, treat as equal. - return Math.abs(a - b) < 0.0001 * Math.max(Math.abs(a), Math.abs(b)); + return Math.abs(a - b) < 0.001 * Math.max(Math.abs(a), Math.abs(b)) + 0.001; } - public static List getResultSetFirstColumnAsString(String queryString, Set errors, - GlobalState state) throws SQLException { + public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, + SQLGlobalState state) throws SQLException { if (state.getOptions().logEachSelect()) { // TODO: refactor me state.getLogger().writeCurrent(queryString); @@ -44,38 +48,38 @@ public static List getResultSetFirstColumnAsString(String queryString, S e.printStackTrace(); } } - QueryAdapter q = new QueryAdapter(queryString, errors); + boolean canonicalizeString = state.getOptions().canonicalizeSqlString(); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, true, canonicalizeString); List resultSet = new ArrayList<>(); - ResultSet result = null; + SQLancerResultSet result = null; try { result = q.executeAndGet(state); if (result == null) { throw new IgnoreMeException(); } while (result.next()) { - resultSet.add(result.getString(1)); + String resultTemp = result.getString(1); + if (resultTemp != null) { + resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); // Remove the trailing zeros as many DBMS treat + // it as non-bugs + } + resultSet.add(resultTemp); } - result.getStatement().close(); } catch (Exception e) { if (e instanceof IgnoreMeException) { throw e; } - if (e instanceof NumberFormatException) { - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/57 - throw new IgnoreMeException(); - } - if (e.getMessage() == null) { - throw new AssertionError(queryString, e); - } - for (String error : errors) { - if (e.getMessage().contains(error)) { + + Throwable current = e; + while (current != null) { + if (current.getMessage() != null && errors.errorIsExpected(current.getMessage())) { throw new IgnoreMeException(); } + current = current.getCause(); } throw new AssertionError(queryString, e); } finally { if (result != null && !result.isClosed()) { - result.getStatement().close(); result.close(); } } @@ -83,42 +87,63 @@ public static List getResultSetFirstColumnAsString(String queryString, S } public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, GlobalState state) { + String originalQueryString, List combinedString, SQLGlobalState state) { if (resultSet.size() != secondResultSet.size()) { - String queryFormatString = "%s; -- cardinality: %d"; + String queryFormatString = "-- %s;" + System.lineSeparator() + "-- cardinality: %d" + + System.lineSeparator(); String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); - String secondQueryString = String.format(queryFormatString, - combinedString.stream().collect(Collectors.joining(";")), secondResultSet.size()); - state.getState().statements.add(new QueryAdapter(firstQueryString)); - state.getState().statements.add(new QueryAdapter(secondQueryString)); - String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", - resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); + String combinedQueryString = String.join(";", combinedString); + String secondQueryString = String.format(queryFormatString, combinedQueryString, secondResultSet.size()); + state.getState().getLocalState() + .log(String.format("%s" + System.lineSeparator() + "%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + resultSet.size(), secondResultSet.size(), originalQueryString, resultSet.size(), + combinedQueryString, secondResultSet.size()); throw new AssertionError(assertionMessage); } Set firstHashSet = new HashSet<>(resultSet); Set secondHashSet = new HashSet<>(secondResultSet); - if (!firstHashSet.equals(secondHashSet)) { + boolean validateResultSizeOnly = state.getOptions().validateResultSizeOnly(); + if (!validateResultSizeOnly && !firstHashSet.equals(secondHashSet)) { Set firstResultSetMisses = new HashSet<>(firstHashSet); firstResultSetMisses.removeAll(secondHashSet); Set secondResultSetMisses = new HashSet<>(secondHashSet); secondResultSetMisses.removeAll(firstHashSet); - String queryFormatString = "%s; -- misses: %s"; + + String queryFormatString = "-- Query: \"%s\"; It misses: \"%s\""; String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); - String secondQueryString = String.format(queryFormatString, - combinedString.stream().collect(Collectors.joining(";")), secondResultSetMisses); - state.getState().statements.add(new QueryAdapter(firstQueryString)); - state.getState().statements.add(new QueryAdapter(secondQueryString)); - String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", - firstQueryString, secondQueryString); + String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), + secondResultSetMisses); + // update the SELECT queries to be logged at the bottom of the error log file + state.getState().getLocalState() + .log(String.format("%s" + System.lineSeparator() + "%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format("The content of the result sets mismatch!" + System.lineSeparator() + + "First query : \"%s\"" + System.lineSeparator() + "Second query: \"%s\"", originalQueryString, + secondQueryString); throw new AssertionError(assertionMessage); } } + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + String originalQueryString, List combinedString, SQLGlobalState state, + UnaryOperator canonicalizationRule) { + // Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to + // both result sets before their comparison. + List canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList()); + List canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule) + .collect(Collectors.toList()); + assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString, + combinedString, state); + } + public static List getCombinedResultSet(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, - Set errors) throws SQLException { + String thirdQueryString, List combinedString, boolean asUnion, SQLGlobalState state, + ExpectedErrors errors) throws SQLException { List secondResultSet; if (asUnion) { String unionString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " @@ -138,8 +163,8 @@ public static List getCombinedResultSet(String firstQueryString, String } public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, GlobalState state, - Set errors) throws SQLException { + String thirdQueryString, List combinedString, boolean asUnion, SQLGlobalState state, + ExpectedErrors errors) throws SQLException { String unionString; if (asUnion) { unionString = firstQueryString + " UNION " + secondQueryString + " UNION " + thirdQueryString; @@ -153,4 +178,20 @@ public static List getCombinedResultSetNoDuplicates(String firstQueryStr return secondResultSet; } + public static String canonicalizeResultValue(String value) { + if (value == null) { + return value; + } + + switch (value) { + case "-0.0": + return "0.0"; + case "-0": + return "0"; + default: + } + + return value; + } + } diff --git a/src/sqlancer/CompositeTestOracle.java b/src/sqlancer/CompositeTestOracle.java deleted file mode 100644 index 9851ee24c..000000000 --- a/src/sqlancer/CompositeTestOracle.java +++ /dev/null @@ -1,23 +0,0 @@ -package sqlancer; - -import java.sql.SQLException; -import java.util.List; - -public class CompositeTestOracle implements TestOracle { - - private final TestOracle[] oracles; - private int i; - - public CompositeTestOracle(List oracles) { - this.oracles = oracles.toArray(new TestOracle[oracles.size()]); - } - - @Override - public void check() throws SQLException { - try { - oracles[i].check(); - } finally { - i = (i + 1) % oracles.length; - } - } -} diff --git a/src/sqlancer/DBMSSpecificOptions.java b/src/sqlancer/DBMSSpecificOptions.java new file mode 100644 index 000000000..4607557d1 --- /dev/null +++ b/src/sqlancer/DBMSSpecificOptions.java @@ -0,0 +1,9 @@ +package sqlancer; + +import java.util.List; + +public interface DBMSSpecificOptions>> { + + List getTestOracleFactory(); + +} diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index ff323f8bf..d169324fa 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -1,18 +1,20 @@ package sqlancer; -import java.io.FileWriter; -import java.sql.Connection; -import java.sql.SQLException; +import sqlancer.common.log.LoggableFactory; -public interface DatabaseProvider, O> { +public interface DatabaseProvider, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> { /** * Gets the the {@link GlobalState} class. + * + * @return the class extending {@link GlobalState} */ Class getGlobalStateClass(); /** * Gets the JCommander option class. + * + * @return the class representing the DBMS-specific options. */ Class getOptionClass(); @@ -22,24 +24,36 @@ public interface DatabaseProvider, O> { * @param globalState * the state created and is valid for this method call. * + * @return Reproducer if a bug is found and a reproducer is available. + * + * @throws Exception + * if creating the database fails. + * */ - void generateAndTestDatabase(G globalState) throws SQLException; - - Connection createDatabase(G globalState) throws SQLException; + Reproducer generateAndTestDatabase(G globalState) throws Exception; /** - * The DBMS name is used to name the log directory and command to test the respective DBMS. + * The experimental feature: Query Plan Guidance. + * + * @param globalState + * the state created and is valid for this method call. + * + * @throws Exception + * if testing fails. + * */ - String getDBMSName(); + void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws Exception; + + C createDatabase(G globalState) throws Exception; - // TODO: remove this /** - * Deprecated method to print the database-specific state, previously used for PQS. + * The DBMS name is used to name the log directory and command to test the respective DBMS. * - * @param writer - * @param state + * @return the DBMS' name */ - void printDatabaseSpecificState(FileWriter writer, StateToReproduce state); + String getDBMSName(); + + LoggableFactory getLoggableFactory(); StateToReproduce getStateToReproduce(String databaseName); diff --git a/src/sqlancer/ExecutionTimer.java b/src/sqlancer/ExecutionTimer.java new file mode 100644 index 000000000..3d88697b3 --- /dev/null +++ b/src/sqlancer/ExecutionTimer.java @@ -0,0 +1,23 @@ +package sqlancer; + +public final class ExecutionTimer { + + private long startTime; + private long endTime; + + public ExecutionTimer start() { + startTime = System.currentTimeMillis(); + return this; + } + + public ExecutionTimer end() { + endTime = System.currentTimeMillis(); + return this; + } + + public String asString() { + long timeMillis = endTime - startTime; + return timeMillis + "ms"; + } + +} diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 16c4e6233..2b93012c2 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,42 +1,37 @@ package sqlancer; -import java.sql.Connection; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; +public abstract class GlobalState, S extends AbstractSchema, C extends SQLancerDBConnection> { -/** - * Represents a global state that is valid for a testing session on a given database. - * - * @param - * the option parameter. - */ -public class GlobalState { - - private Connection con; + protected C databaseConnection; private Randomly r; private MainOptions options; - private O dmbsSpecificOptions; - private StateLogger logger; + private O dbmsSpecificOptions; + private S schema; + private Main.StateLogger logger; private StateToReproduce state; - private QueryManager manager; + private Main.QueryManager manager; private String databaseName; - public void setConnection(Connection con) { - this.con = con; + public void setConnection(C con) { + this.databaseConnection = con; } - @SuppressWarnings("unchecked") - public void setDmbsSpecificOptions(Object dmbsSpecificOptions) { - this.dmbsSpecificOptions = (O) dmbsSpecificOptions; + public C getConnection() { + return databaseConnection; } - public O getDmbsSpecificOptions() { - return dmbsSpecificOptions; + @SuppressWarnings("unchecked") + public void setDbmsSpecificOptions(Object dbmsSpecificOptions) { + this.dbmsSpecificOptions = (O) dbmsSpecificOptions; } - public Connection getConnection() { - return con; + public O getDbmsSpecificOptions() { + return dbmsSpecificOptions; } public void setRandomly(Randomly r) { @@ -55,11 +50,11 @@ public void setMainOptions(MainOptions options) { this.options = options; } - public void setStateLogger(StateLogger logger) { + public void setStateLogger(Main.StateLogger logger) { this.logger = logger; } - public StateLogger getLogger() { + public Main.StateLogger getLogger() { return logger; } @@ -71,11 +66,11 @@ public StateToReproduce getState() { return state; } - public QueryManager getManager() { + public Main.QueryManager getManager() { return manager; } - public void setManager(QueryManager manager) { + public void setManager(Main.QueryManager manager) { this.manager = manager; } @@ -87,4 +82,72 @@ public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } + private ExecutionTimer executePrologue(Query q) throws Exception { + boolean logExecutionTime = getOptions().logExecutionTime(); + ExecutionTimer timer = null; + if (logExecutionTime) { + timer = new ExecutionTimer().start(); + } + if (getOptions().printAllStatements()) { + System.out.println(q.getLogString()); + } + if (getOptions().logEachSelect()) { + if (logExecutionTime) { + getLogger().writeCurrentNoLineBreak(q.getLogString()); + } else { + getLogger().writeCurrent(q.getLogString()); + } + } + return timer; + } + + protected abstract void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception; + + public boolean executeStatement(Query q, String... fills) throws Exception { + ExecutionTimer timer = executePrologue(q); + boolean success = manager.execute(q, fills); + executeEpilogue(q, success, timer); + return success; + } + + public SQLancerResultSet executeStatementAndGet(Query q, String... fills) throws Exception { + ExecutionTimer timer = executePrologue(q); + SQLancerResultSet result = manager.executeAndGet(q, fills); + boolean success = result != null; + if (success) { + result.registerEpilogue(() -> { + try { + executeEpilogue(q, success, timer); + } catch (Exception e) { + throw new AssertionError(e); + } + }); + } + return result; + } + + public S getSchema() { + if (schema == null) { + try { + updateSchema(); + } catch (Exception e) { + throw new AssertionError(e.getMessage()); + } + } + return schema; + } + + protected void setSchema(S schema) { + this.schema = schema; + } + + public void updateSchema() throws Exception { + setSchema(readSchema()); + for (AbstractTable table : schema.getDatabaseTables()) { + table.recomputeCount(); + } + } + + protected abstract S readSchema() throws Exception; + } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index ae8f874b7..47ba2aedf 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -3,13 +3,9 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; import java.io.Writer; import java.nio.file.Files; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.SQLFeatureNotSupportedException; +import java.nio.file.Path; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -17,23 +13,41 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.ServiceLoader; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; +import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.cockroachdb.CockroachDBProvider; +import sqlancer.common.log.Loggable; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendProvider; +import sqlancer.doris.DorisProvider; import sqlancer.duckdb.DuckDBProvider; +import sqlancer.h2.H2Provider; +import sqlancer.hive.HiveProvider; +import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.mariadb.MariaDBProvider; +import sqlancer.materialize.MaterializeProvider; import sqlancer.mysql.MySQLProvider; +import sqlancer.oceanbase.OceanBaseProvider; import sqlancer.postgres.PostgresProvider; +import sqlancer.presto.PrestoProvider; +import sqlancer.questdb.QuestDBProvider; +import sqlancer.spark.SparkProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; +import sqlancer.yugabyte.ycql.YCQLProvider; +import sqlancer.yugabyte.ysql.YSQLProvider; public final class Main { @@ -42,10 +56,11 @@ public final class Main { public static volatile AtomicLong nrDatabases = new AtomicLong(); public static volatile AtomicLong nrSuccessfulActions = new AtomicLong(); public static volatile AtomicLong nrUnsuccessfulActions = new AtomicLong(); - static int threadsShutdown; + public static volatile AtomicLong threadsShutdown = new AtomicLong(); + static boolean progressMonitorStarted; static { - System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "ERROR"); + System.setProperty(org.slf4j.simple.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "ERROR"); if (!LOG_DIRECTORY.exists()) { LOG_DIRECTORY.mkdir(); } @@ -58,11 +73,23 @@ public static final class StateLogger { private final File loggerFile; private File curFile; + private File queryPlanFile; + private File reduceFile; private FileWriter logFileWriter; public FileWriter currentFileWriter; + private FileWriter queryPlanFileWriter; + private FileWriter reduceFileWriter; + private Path reproduceFilePath; + private List> reduceSetupStatements; + private String reduceBugInformation; + private int nrReductionAttempts; + private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; - private final DatabaseProvider provider; + private final boolean logQueryPlan; + + private final boolean useReducer; + private final DatabaseProvider databaseProvider; private static final class AlsoWriteToConsoleFileWriter extends FileWriter { @@ -83,8 +110,7 @@ public void write(String str) throws IOException { } } - public StateLogger(String databaseName, DatabaseProvider provider, MainOptions options) { - this.provider = provider; + public StateLogger(String databaseName, DatabaseProvider provider, MainOptions options) { File dir = new File(LOG_DIRECTORY, provider.getDBMSName()); if (dir.exists() && !dir.isDirectory()) { throw new AssertionError(dir); @@ -95,27 +121,49 @@ public StateLogger(String databaseName, DatabaseProvider provider, MainOpt if (logEachSelect) { curFile = new File(dir, databaseName + "-cur.log"); } + logQueryPlan = options.logQueryPlan(); + if (logQueryPlan) { + queryPlanFile = new File(dir, databaseName + "-plan.log"); + } + this.useReducer = options.useReducer(); + if (useReducer) { + File reduceFileDir = new File(dir, "reduce"); + if (!reduceFileDir.exists()) { + reduceFileDir.mkdir(); + } + this.reduceFile = new File(reduceFileDir, databaseName + "-reduce.log"); + } + if (options.serializeReproduceState()) { + File reproduceFileDir = new File(dir, "reproduce"); + if (!reproduceFileDir.exists()) { + reproduceFileDir.mkdir(); + } + reproduceFilePath = new File(reproduceFileDir, databaseName + ".ser").toPath(); + } + this.databaseProvider = provider; } - private synchronized void ensureExistsAndIsEmpty(File dir, DatabaseProvider provider) { + private void ensureExistsAndIsEmpty(File dir, DatabaseProvider provider) { if (INITIALIZED_PROVIDER_NAMES.contains(provider.getDBMSName())) { return; } - if (!dir.exists()) { - try { - Files.createDirectories(dir.toPath()); - } catch (IOException e) { - throw new AssertionError(e); + synchronized (INITIALIZED_PROVIDER_NAMES) { + if (!dir.exists()) { + try { + Files.createDirectories(dir.toPath()); + } catch (IOException e) { + throw new AssertionError(e); + } } - } - File[] listFiles = dir.listFiles(); - assert listFiles != null : "directory was just created, so it should exist"; - for (File file : listFiles) { - if (!file.isDirectory()) { - file.delete(); + File[] listFiles = dir.listFiles(); + assert listFiles != null : "directory was just created, so it should exist"; + for (File file : listFiles) { + if (!file.isDirectory()) { + file.delete(); + } } + INITIALIZED_PROVIDER_NAMES.add(provider.getDBMSName()); } - INITIALIZED_PROVIDER_NAMES.add(provider.getDBMSName()); } private FileWriter getLogFileWriter() { @@ -143,6 +191,34 @@ public FileWriter getCurrentFileWriter() { return currentFileWriter; } + public FileWriter getQueryPlanFileWriter() { + if (!logQueryPlan) { + throw new UnsupportedOperationException(); + } + if (queryPlanFileWriter == null) { + try { + queryPlanFileWriter = new FileWriter(queryPlanFile, true); + } catch (IOException e) { + throw new AssertionError(e); + } + } + return queryPlanFileWriter; + } + + public FileWriter getReduceFileWriter() { + if (!useReducer) { + throw new UnsupportedOperationException(); + } + if (reduceFileWriter == null) { + try { + reduceFileWriter = new FileWriter(reduceFile, false); + } catch (IOException e) { + throw new AssertionError(e); + } + } + return reduceFileWriter; + } + public void writeCurrent(StateToReproduce state) { if (!logEachSelect) { throw new UnsupportedOperationException(); @@ -156,33 +232,92 @@ public void writeCurrent(StateToReproduce state) { } } - public void writeCurrent(String queryString) { + public void writeCurrent(String input) { + write(databaseProvider.getLoggableFactory().createLoggable(input)); + } + + public void writeCurrentNoLineBreak(String input) { + write(databaseProvider.getLoggableFactory().createLoggableWithNoLinebreak(input)); + } + + private void write(Loggable loggable) { if (!logEachSelect) { throw new UnsupportedOperationException(); } try { - getCurrentFileWriter().write(queryString + ";\n"); + getCurrentFileWriter().write(loggable.getLogString()); + currentFileWriter.flush(); } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + throw new AssertionError(); } } - public void logRowNotFound(StateToReproduce state) { - printState(getLogFileWriter(), state); + public void writeQueryPlan(String queryPlan) { + if (!logQueryPlan) { + throw new UnsupportedOperationException(); + } try { - getLogFileWriter().flush(); + getQueryPlanFileWriter().append(removeNamesFromQueryPlans(queryPlan)); + queryPlanFileWriter.flush(); + } catch (IOException e) { + throw new AssertionError(); + } + } + + public void setReductionContext(List> setupStatements, String bugInformation) { + this.reduceSetupStatements = setupStatements; + this.reduceBugInformation = bugInformation; + } + + public void logReduced(StateToReproduce state) { + nrReductionAttempts++; + logReduced(state, "Reduction attempt " + nrReductionAttempts + + ": the bug was still triggered with the following statements"); + } + + public void logReduced(StateToReproduce state, String description) { + FileWriter reduceFileWriter = getReduceFileWriter(); + + StringBuilder sb = new StringBuilder(); + sb.append("-- ").append(description).append(System.lineSeparator()); + if (reduceSetupStatements != null && !reduceSetupStatements.isEmpty()) { + appendStatements(sb, reduceSetupStatements); + // e.g. DROP DATABASE IF EXISTS db; CREATE DATABASE db; USE db; + // these statements are executed at the start of every test case and are never reduced + } + appendStatements(sb, state.getStatements()); + if (reduceBugInformation != null) { + sb.append(reduceBugInformation); + } + sb.append(System.lineSeparator()); + try { + reduceFileWriter.write(sb.toString()); + } catch (IOException e) { throw new AssertionError(e); + } finally { + try { + reduceFileWriter.flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + } + + private void appendStatements(StringBuilder sb, List> statements) { + for (Query s : statements) { + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); } } public void logException(Throwable reduce, StateToReproduce state) { - String stackTrace = getStackTrace(reduce); + Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); try { - logFileWriter2.write(stackTrace); + logFileWriter2.write(stackTrace.getLogString()); printState(logFileWriter2, state); } catch (IOException e) { throw new AssertionError(e); @@ -190,100 +325,105 @@ public void logException(Throwable reduce, StateToReproduce state) { try { logFileWriter2.flush(); } catch (IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } } - private String getStackTrace(Throwable e1) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - e1.printStackTrace(pw); - return "--" + sw.toString().replace("\n", "\n--"); + private Loggable getStackTrace(Throwable e1) { + return databaseProvider.getLoggableFactory().convertStacktraceToLoggable(e1); } private void printState(FileWriter writer, StateToReproduce state) { StringBuilder sb = new StringBuilder(); - DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); - Date date = new Date(); - sb.append("-- Time: " + dateFormat.format(date) + "\n"); - sb.append("-- Database: " + state.getDatabaseName() + "\n"); - sb.append("-- Database version: " + state.getDatabaseVersion() + "\n"); - sb.append("-- seed value: " + state.getSeedValue() + "\n"); - for (Query s : state.getStatements()) { - if (s.getQueryString().endsWith(";")) { - sb.append(s.getQueryString()); - } else { - sb.append(s.getQueryString() + ";"); - } - sb.append('\n'); - } - if (state.getQueryString() != null) { - sb.append(state.getQueryString() + ";\n"); + + sb.append(databaseProvider.getLoggableFactory() + .getInfo(state.getDatabaseName(), state.getDatabaseVersion(), state.getSeedValue()).getLogString()); + + for (Query s : state.getStatements()) { + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); } try { writer.write(sb.toString()); } catch (IOException e) { throw new AssertionError(e); } - provider.printDatabaseSpecificState(writer, state); } + private String removeNamesFromQueryPlans(String queryPlan) { + String result = queryPlan; + result = result.replaceAll("t[0-9]+", "t0"); // Avoid duplicate tables + result = result.replaceAll("v[0-9]+", "v0"); // Avoid duplicate views + result = result.replaceAll("i[0-9]+", "i0"); // Avoid duplicate indexes + return result + "\n"; + } + + public Path getReproduceFilePath() { + return reproduceFilePath; + } } - public static class QueryManager { + public static class QueryManager { - private final GlobalState globalState; + private final GlobalState globalState; - QueryManager(GlobalState globalState) { + QueryManager(GlobalState globalState) { this.globalState = globalState; } - public boolean execute(Query q) throws SQLException { - globalState.getState().statements.add(q); - boolean success = q.execute(globalState); + public boolean execute(Query q, String... fills) throws Exception { + boolean success; + success = q.execute(globalState, fills); Main.nrSuccessfulActions.addAndGet(1); + if (globalState.getOptions().loggerPrintFailed() || success) { + globalState.getState().logStatement(q); + } return success; } + public SQLancerResultSet executeAndGet(Query q, String... fills) throws Exception { + globalState.getState().logStatement(q); + SQLancerResultSet result; + result = q.executeAndGet(globalState, fills); + Main.nrSuccessfulActions.addAndGet(1); + return result; + } + public void incrementSelectQueryCount() { Main.nrQueries.addAndGet(1); } + public Long getSelectQueryCount() { + return Main.nrQueries.get(); + } + public void incrementCreateDatabase() { Main.nrDatabases.addAndGet(1); } } - public static void printArray(Object... arr) { - for (Object o : arr) { - System.out.println(o); - } - } - public static void main(String[] args) { System.exit(executeMain(args)); } - public static class DBMSExecutor, O> { + public static class DBMSExecutor, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> { - private final DatabaseProvider provider; + private final DatabaseProvider provider; private final MainOptions options; private final O command; private final String databaseName; - private final long seed; private StateLogger logger; private StateToReproduce stateToRepro; + private final Randomly r; - public DBMSExecutor(DatabaseProvider provider, MainOptions options, O dbmsSpecificOptions, - String databaseName, long seed) { + public DBMSExecutor(DatabaseProvider provider, MainOptions options, O dbmsSpecificOptions, + String databaseName, Randomly r) { this.provider = provider; this.options = options; this.databaseName = databaseName; - this.seed = seed; this.command = dbmsSpecificOptions; + this.r = r; } private G createGlobalState() { @@ -298,32 +438,136 @@ public O getCommand() { return command; } - public void run() throws SQLException { + public void testConnection() throws Exception { + G state = getInitializedGlobalState(options.getRandomSeed()); + try (SQLancerDBConnection con = provider.createDatabase(state)) { + return; + } + } + + public void run() throws Exception { G state = createGlobalState(); stateToRepro = provider.getStateToReproduce(databaseName); - stateToRepro.seedValue = seed; + stateToRepro.seedValue = r.getSeed(); state.setState(stateToRepro); logger = new StateLogger(databaseName, provider, options); - Randomly r = new Randomly(seed); state.setRandomly(r); state.setDatabaseName(databaseName); state.setMainOptions(options); - state.setDmbsSpecificOptions(command); - try (Connection con = provider.createDatabase(state)) { - QueryManager manager = new QueryManager(state); + state.setDbmsSpecificOptions(command); + try (C con = provider.createDatabase(state)) { + QueryManager manager = new QueryManager<>(state); try { - java.sql.DatabaseMetaData meta = con.getMetaData(); - stateToRepro.databaseVersion = meta.getDatabaseProductVersion(); - } catch (SQLFeatureNotSupportedException e) { + stateToRepro.databaseVersion = con.getDatabaseVersion(); + } catch (Exception e) { // ignore } state.setConnection(con); state.setStateLogger(logger); state.setManager(manager); - provider.generateAndTestDatabase(state); + if (options.logEachSelect()) { + logger.writeCurrent(state.getState()); + } + // statements logged so far stem from the database setup (e.g., DROP DATABASE IF + // EXISTS, CREATE DATABASE, USE), performed by createDatabase + int nrSetupStatements = stateToRepro.getStatements().size(); + Reproducer reproducer = null; + if (options.enableQPG()) { + provider.generateAndTestDatabaseWithQueryPlanGuidance(state); + } else { + reproducer = provider.generateAndTestDatabase(state); + } + try { + logger.getCurrentFileWriter().close(); + logger.currentFileWriter = null; + } catch (IOException e) { + throw new AssertionError(e); + } + + if (options.serializeReproduceState() && reproducer != null) { + stateToRepro.serialize(logger.getReproduceFilePath()); + } + if (options.reduceAST() && !options.useReducer()) { + throw new AssertionError("To reduce AST, use-reducer option must be enabled first"); + } + if (options.useReducer()) { + if (reproducer == null) { + logger.getReduceFileWriter().write("current oracle does not support experimental reducer."); + throw new IgnoreMeException(); + } + + // reduce only the generation statements: the database setup (logged by + // createDatabase) is re-executed by the reducers for every candidate, and the + // oracle queries (logged by the oracle's local state) by the reproducer + List> allStatements = new ArrayList<>(stateToRepro.getStatements()); + List> setupStatements = new ArrayList<>(allStatements.subList(0, nrSetupStatements)); + List> oracleQueryStatements = stateToRepro.getLocalState() == null ? new ArrayList<>() + : new ArrayList<>(stateToRepro.getLocalState().getStatements()); + stateToRepro.setStatements(new ArrayList<>(allStatements.subList(nrSetupStatements, + allStatements.size() - oracleQueryStatements.size()))); + + G newGlobalState = createGlobalState(); + newGlobalState.setState(stateToRepro); + newGlobalState.setRandomly(r); + newGlobalState.setDatabaseName(databaseName); + newGlobalState.setMainOptions(options); + newGlobalState.setDbmsSpecificOptions(command); + QueryManager newManager = new QueryManager<>(newGlobalState); + newGlobalState.setStateLogger(new StateLogger(databaseName, provider, options)); + newGlobalState.setManager(newManager); + newGlobalState.getLogger().setReductionContext(setupStatements, reproducer.getBugInformation()); + + Reducer reducer = new StatementReducer<>(provider); + reducer.reduce(state, reproducer, newGlobalState); + + if (options.reduceAST()) { + Reducer astBasedReducer = new ASTBasedReducer<>(provider); + astBasedReducer.reduce(state, reproducer, newGlobalState); + } + + // reassemble the statements so that the main log looks like one produced + // without the reducer, with the generation statements replaced by the reduced + // ones and the oracle queries at the end + List> finalStatements = new ArrayList<>(setupStatements); + finalStatements.addAll(stateToRepro.getStatements()); + finalStatements.addAll(oracleQueryStatements); + stateToRepro.setStatements(finalStatements); + String bugInformation = reproducer.getBugInformation(); + if (bugInformation != null) { + for (String line : bugInformation.split(System.lineSeparator())) { + stateToRepro.logStatement(line); + } + } + + StateLogger reduceLogger = newGlobalState.getLogger(); + if (reduceLogger.reduceFileWriter != null) { + try { + reduceLogger.reduceFileWriter.close(); + reduceLogger.reduceFileWriter = null; + } catch (IOException e) { + throw new AssertionError(e); + } + } + + throw new AssertionError("Found a potential bug, please check reducer log for detail."); + } } } + private G getInitializedGlobalState(long seed) { + G state = createGlobalState(); + stateToRepro = provider.getStateToReproduce(databaseName); + stateToRepro.seedValue = seed; + state.setState(stateToRepro); + logger = new StateLogger(databaseName, provider, options); + Randomly r = new Randomly(seed); + state.setRandomly(r); + state.setDatabaseName(databaseName); + state.setMainOptions(options); + state.setDbmsSpecificOptions(command); + return state; + } + public StateLogger getLogger() { return logger; } @@ -333,13 +577,13 @@ public StateToReproduce getStateToReproduce() { } } - public static class DBMSExecutorFactory, O> { + public static class DBMSExecutorFactory, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> { - private final DatabaseProvider provider; + private final DatabaseProvider provider; private final MainOptions options; private final O command; - public DBMSExecutorFactory(DatabaseProvider provider, MainOptions options) { + public DBMSExecutorFactory(DatabaseProvider provider, MainOptions options) { this.provider = provider; this.options = options; this.command = createCommand(); @@ -358,54 +602,93 @@ public O getCommand() { } @SuppressWarnings("unchecked") - public DBMSExecutor getDBMSExecutor(String databaseName, long seed) { + public DBMSExecutor getDBMSExecutor(String databaseName, Randomly r) { try { - return new DBMSExecutor(provider.getClass().getDeclaredConstructor().newInstance(), options, - command, databaseName, seed); + return new DBMSExecutor(provider.getClass().getDeclaredConstructor().newInstance(), options, + command, databaseName, r); } catch (Exception e) { throw new AssertionError(e); } } + public DatabaseProvider getProvider() { + return provider; + } + } public static int executeMain(String... args) throws AssertionError { - List> providers = getDBMSProviders(); - Map> nameToProvider = new HashMap<>(); + List> providers = getDBMSProviders(); + Map> nameToProvider = new HashMap<>(); MainOptions options = new MainOptions(); Builder commandBuilder = JCommander.newBuilder().addObject(options); - for (DatabaseProvider provider : providers) { + for (DatabaseProvider provider : providers) { String name = provider.getDBMSName(); - if (!name.toLowerCase().equals(name)) { - throw new AssertionError(name + " should be in lowercase!"); - } - DBMSExecutorFactory executorFactory = new DBMSExecutorFactory<>(provider, options); + DBMSExecutorFactory executorFactory = new DBMSExecutorFactory<>(provider, options); commandBuilder = commandBuilder.addCommand(name, executorFactory.getCommand()); nameToProvider.put(name, executorFactory); } JCommander jc = commandBuilder.programName("SQLancer").build(); jc.parse(args); - if (jc.getParsedCommand() == null) { + if (jc.getParsedCommand() == null || options.isHelp()) { jc.usage(); return options.getErrorExitCode(); } + Randomly.initialize(options); if (options.printProgressInformation()) { startProgressMonitor(); + if (options.printProgressSummary()) { + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + + @Override + public void run() { + System.out.println("Overall execution statistics"); + System.out.println("============================"); + System.out.println(formatInteger(nrQueries.get()) + " queries"); + System.out.println(formatInteger(nrDatabases.get()) + " databases"); + System.out.println( + formatInteger(nrSuccessfulActions.get()) + " successfully-executed statements"); + System.out.println( + formatInteger(nrUnsuccessfulActions.get()) + " unsuccessfully-executed statements"); + } + + private String formatInteger(long intValue) { + if (intValue > 1000) { + return String.format("%,9dk", intValue / 1000); + } else { + return String.format("%,10d", intValue); + } + } + })); + } } ExecutorService execService = Executors.newFixedThreadPool(options.getNumberConcurrentThreads()); - DBMSExecutorFactory executorFactory = nameToProvider.get(jc.getParsedCommand()); + DBMSExecutorFactory executorFactory = nameToProvider.get(jc.getParsedCommand()); + + if (options.performConnectionTest()) { + try { + executorFactory.getDBMSExecutor(options.getDatabasePrefix() + "connectiontest", new Randomly()) + .testConnection(); + } catch (Exception e) { + System.err.println( + "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username, password, host and port, you can use the --username, --password, --host and --port options.\n\n"); + e.printStackTrace(); + return options.getErrorExitCode(); + } + } + final AtomicBoolean someOneFails = new AtomicBoolean(false); + for (int i = 0; i < options.getTotalNumberTries(); i++) { - final String databaseName = "database" + i; + final String databaseName = options.getDatabasePrefix() + i; final long seed; if (options.getRandomSeed() == -1) { seed = System.currentTimeMillis() + i; } else { seed = options.getRandomSeed() + i; } - execService.execute(new Runnable() { @Override @@ -415,33 +698,53 @@ public void run() { } private void runThread(final String databaseName) { - while (true) { - DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, seed); + Randomly r = new Randomly(seed); + try { + int maxNrDbs = options.getMaxGeneratedDatabases(); + // run without a limit if maxNrDbs == -1 + for (int i = 0; i < maxNrDbs || maxNrDbs == -1; i++) { + Boolean continueRunning = run(options, execService, executorFactory, r, databaseName); + if (!continueRunning) { + someOneFails.set(true); + break; + } + } + } finally { + threadsShutdown.addAndGet(1); + if (threadsShutdown.get() == options.getTotalNumberTries()) { + execService.shutdown(); + } + } + } + + private boolean run(MainOptions options, ExecutorService execService, + DBMSExecutorFactory executorFactory, Randomly r, final String databaseName) { + DBMSExecutor executor = executorFactory.getDBMSExecutor(databaseName, r); + try { + executor.run(); + return true; + } catch (IgnoreMeException e) { + return true; + } catch (Throwable reduce) { + reduce.printStackTrace(); + executor.getStateToReproduce().exception = reduce.getMessage(); + executor.getLogger().logFileWriter = null; + executor.getLogger().logException(reduce, executor.getStateToReproduce()); + if (options.serializeReproduceState()) { + executor.getStateToReproduce().logStatement(reduce.getMessage()); // add the error statement + executor.getStateToReproduce().serialize(executor.getLogger().getReproduceFilePath()); + } + return false; + } finally { try { - executor.run(); - } catch (IgnoreMeException e) { - continue; - } catch (Throwable reduce) { - reduce.printStackTrace(); - executor.getStateToReproduce().exception = reduce.getMessage(); - executor.getLogger().logFileWriter = null; - executor.getLogger().logException(reduce, executor.getStateToReproduce()); - threadsShutdown++; - break; - } finally { - try { - if (options.logEachSelect()) { - if (executor.getLogger().currentFileWriter != null) { - executor.getLogger().currentFileWriter.close(); - } - executor.getLogger().currentFileWriter = null; + if (options.logEachSelect()) { + if (executor.getLogger().currentFileWriter != null) { + executor.getLogger().currentFileWriter.close(); } - } catch (IOException e) { - e.printStackTrace(); - } - if (threadsShutdown == options.getTotalNumberTries()) { - execService.shutdown(); + executor.getLogger().currentFileWriter = null; } + } catch (IOException e) { + e.printStackTrace(); } } } @@ -456,23 +759,68 @@ private void runThread(final String databaseName) { } catch (InterruptedException e) { e.printStackTrace(); } - return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); + + return someOneFails.get() ? options.getErrorExitCode() : 0; } - static List> getDBMSProviders() { - List> providers = new ArrayList<>(); - providers.add(new SQLite3Provider()); - providers.add(new CockroachDBProvider()); - providers.add(new MySQLProvider()); - providers.add(new MariaDBProvider()); - providers.add(new TiDBProvider()); - providers.add(new PostgresProvider()); - providers.add(new ClickHouseProvider()); - providers.add(new DuckDBProvider()); + /** + * To register a new provider, it is necessary to implement the DatabaseProvider interface and add an additional + * configuration file, see https://docs.oracle.com/javase/9/docs/api/java/util/ServiceLoader.html. Currently, we use + * an @AutoService annotation to create the configuration file automatically. This allows SQLancer to pick up + * providers in other JARs on the classpath. + * + * @return The list of service providers on the classpath + */ + static List> getDBMSProviders() { + List> providers = new ArrayList<>(); + @SuppressWarnings("rawtypes") + ServiceLoader loader = ServiceLoader.load(DatabaseProvider.class); + for (DatabaseProvider provider : loader) { + providers.add(provider); + } + checkForIssue799(providers); return providers; } - private static void startProgressMonitor() { + // see https://github.com/sqlancer/sqlancer/issues/799 + private static void checkForIssue799(List> providers) { + if (providers.isEmpty()) { + System.err.println( + "No DBMS implementations (i.e., instantiations of the DatabaseProvider class) were found. You likely ran into an issue described in https://github.com/sqlancer/sqlancer/issues/799. As a workaround, I now statically load all supported providers as of June 7, 2023."); + providers.add(new CitusProvider()); + providers.add(new ClickHouseProvider()); + providers.add(new CockroachDBProvider()); + providers.add(new DatabendProvider()); + providers.add(new DorisProvider()); + providers.add(new DuckDBProvider()); + providers.add(new H2Provider()); + providers.add(new HiveProvider()); + providers.add(new SparkProvider()); + providers.add(new HSQLDBProvider()); + providers.add(new MariaDBProvider()); + providers.add(new MaterializeProvider()); + providers.add(new MySQLProvider()); + providers.add(new OceanBaseProvider()); + providers.add(new PrestoProvider()); + providers.add(new PostgresProvider()); + providers.add(new QuestDBProvider()); + providers.add(new SQLite3Provider()); + providers.add(new TiDBProvider()); + providers.add(new YCQLProvider()); + providers.add(new YSQLProvider()); + } + } + + private static synchronized void startProgressMonitor() { + if (progressMonitorStarted) { + /* + * it might be already started if, for example, the main method is called multiple times in a test (see + * https://github.com/sqlancer/sqlancer/issues/90). + */ + return; + } else { + progressMonitorStarted = true; + } final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); scheduler.scheduleAtFixedRate(new Runnable() { @@ -500,7 +848,7 @@ public void run() { System.out.println(String.format( "[%s] Executed %d queries (%d queries/s; %.2f/s dbs, successful statements: %2d%%). Threads shut down: %d.", dateFormat.format(date), currentNrQueries, (int) throughput, throughputDbs, - successfulStatementsRatio, threadsShutdown)); + successfulStatementsRatio, threadsShutdown.get())); timeMillis = System.currentTimeMillis(); lastNrQueries = currentNrQueries; lastNrDbs = currentNrDbs; diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 6eeac6440..25b769312 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -1,11 +1,20 @@ package sqlancer; -import com.beust.jcommander.IStringConverter; +import java.util.Objects; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import sqlancer.Randomly.StringGenerationStrategy; + @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { + public static final int NO_SET_PORT = -1; + public static final int NO_REDUCE_LIMIT = -1; + public static final MainOptions DEFAULT_OPTIONS = new MainOptions(); + + @Parameter(names = { "--help", "-h" }, description = "Lists all supported options and commands", help = true) + private boolean help; // NOPMD @Parameter(names = { "--num-threads" }, description = "How many threads should run concurrently to test separate databases") @@ -36,21 +45,111 @@ public class MainOptions { @Parameter(names = "--log-each-select", description = "Logs every statement issued", arity = 1) private boolean logEachSelect = true; // NOPMD + @Parameter(names = "--log-execution-time", description = "Logs the execution time of each statement (requires --log-each-select to be enabled)", arity = 1) + private boolean logExecutionTime = true; // NOPMD + + @Parameter(names = "--print-failed", description = "Logs failed insert, create and other statements without results", arity = 1) + private boolean loggerPrintFailed = true; // NOPMD + + @Parameter(names = "--qpg-enable", description = "Enable the experimental feature Query Plan Guidance (QPG)", arity = 1) + private boolean enableQPG; + + @Parameter(names = "--qpg-log-query-plan", description = "Logs the query plans of each query (requires --qpg-enable)", arity = 1) + private boolean logQueryPlan; + + @Parameter(names = "--qpg-max-interval", description = "The maximum number of iterations to mutate tables if no new query plans (requires --qpg-enable)") + private static int qpgMaxInterval = 1000; + + @Parameter(names = "--qpg-reward-weight", description = "The weight (0-1) of last reward when updating weighted average reward. A higher value denotes average reward is more affected by the last reward (requires --qpg-enable)") + private static double qpgk = 0.25; + + @Parameter(names = "--qpg-selection-probability", description = "The probability (0-1) of the random selection of mutators. A higher value (>0.5) favors exploration over exploitation. (requires --qpg-enable)") + private static double qpgProbability = 0.7; + @Parameter(names = "--username", description = "The user name used to log into the DBMS") private String userName = "sqlancer"; // NOPMD @Parameter(names = "--password", description = "The password used to log into the DBMS") private String password = "sqlancer"; // NOPMD + @Parameter(names = "--host", description = "The host used to log into the DBMS") + private String host = null; // NOPMD + + @Parameter(names = "--port", description = "The port used to log into the DBMS") + private int port = MainOptions.NO_SET_PORT; // NOPMD + @Parameter(names = "--print-progress-information", description = "Whether to print progress information such as the number of databases generated or queries issued", arity = 1) private boolean printProgressInformation = true; // NOPMD + @Parameter(names = "--print-progress-summary", description = "Whether to print an execution summary when exiting SQLancer", arity = 1) + private boolean printProgressSummary; // NOPMD + @Parameter(names = "--timeout-seconds", description = "The timeout in seconds") private int timeoutSeconds = -1; // NOPMD + @Parameter(names = "--max-generated-databases", description = "The maximum number of databases that are generated by each thread") + private int maxGeneratedDatabases = -1; // NOPMD + @Parameter(names = "--exit-code-error", description = "The exit code that should be returned when an error is encountered (or a bug is found)") private int errorExitCode = -1; // NOPMD + @Parameter(names = "--print-statements", description = "Print all statements to stdout, before they are sent to the DBMS (not yet implemented for all oracles)", arity = 1) + private boolean printStatements; // NOPMD + + @Parameter(names = "--print-succeeding-statements", description = "Print statements that are successfully processed by the DBMS to stdout (not yet implemented for all oracles)", arity = 1) + private boolean printSucceedingStatements; // NOPMD + + @Parameter(names = "--test-only-nonempty-tables", description = "Test only databases each of whose tables contain at least a single row", arity = 1) + private boolean testOnlyWithMoreThanZeroRows; // NOPMD + + @Parameter(names = "--pqs-test-aggregates", description = "Partially test aggregate functions when all tables contain only a single row.", arity = 1) + private boolean testAggregateFunctions; // NOPMD + + @Parameter(names = "--random-string-generation", description = "Select the random-string eneration approach") + private StringGenerationStrategy randomStringGenerationStrategy = StringGenerationStrategy.SOPHISTICATED; // NOPMD + + @Parameter(names = "--string-constant-max-length", description = "Specify the maximum-length of generated string constants") + private int maxStringConstantLength = 10; // NOPMD + + @Parameter(names = "--use-constant-caching", description = "Specifies whether constants should be cached and re-used with a certain probability", arity = 1) + private boolean useConstantCaching = true; // NOPMD + + @Parameter(names = "--use-connection-test", description = "Test whether the DBMS is accessible before trying to connect using multiple threads", arity = 1) + private boolean useConnectionTest = true; // NOPMD + + @Parameter(names = "--constant-cache-size", description = "Specifies the size of the constant cache. This option only takes effect when constant caching is enabled") + private int constantCacheSize = 100; // NOPMD + + @Parameter(names = "--database-prefix", description = "The prefix used for each database created") + private String databasePrefix = "database"; // NOPMD + + @Parameter(names = "--serialize-reproduce-state", description = "Serialize the state to reproduce") + private boolean serializeReproduceState = false; // NOPMD + + @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer. Implemented for TLP WHERE and NoREC only") + private boolean useReducer = false; // NOPMD + + @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL Perform AST reduction after statement reduction") + private boolean reduceAST = false; // NOPMD + + @Parameter(names = "--statement-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the statement reducer will do") + private long maxStatementReduceSteps = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--statement-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the AST-based reducer will do") + private long maxASTReduceTime = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--ast-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the AST-based reducer will do") + private long maxASTReduceSteps = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--ast-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") + private long maxStatementReduceTime = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--validate-result-size-only", description = "Should validate result size only and skip comparing content of the result set ", arity = 1) + private boolean validateResultSizeOnly = false; // NOPMD + + @Parameter(names = "--canonicalize-sql-strings", description = "Should canonicalize query string (add ';' at the end", arity = 1) + private boolean canonicalizeSqlString = true; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -67,6 +166,51 @@ public boolean logEachSelect() { return logEachSelect; } + public boolean printAllStatements() { + if (printSucceedingStatements && printStatements) { + throw new AssertionError(); + } + return printStatements; + } + + public boolean printSucceedingStatements() { + if (printStatements && printSucceedingStatements) { + throw new AssertionError(); + } + return printSucceedingStatements; + } + + public boolean logExecutionTime() { + if (!logEachSelect) { + throw new AssertionError(); + } + return logExecutionTime; + } + + public boolean loggerPrintFailed() { + return loggerPrintFailed; + } + + public boolean logQueryPlan() { + return logQueryPlan; + } + + public boolean enableQPG() { + return enableQPG; + } + + public int getQPGMaxMutationInterval() { + return qpgMaxInterval; + } + + public double getQPGk() { + return qpgk; + } + + public double getQPGProbability() { + return qpgProbability; + } + public int getNrQueries() { return nrQueries; } @@ -79,10 +223,6 @@ public int getNrStatementRetryCount() { return nrStatementRetryCount; } - public enum DBMS { - MariaDB, SQLite3, MySQL, PostgreSQL, TDEngine, CockroachDB, TiDB, ClickHouse - } - public String getUserName() { return userName; } @@ -91,21 +231,30 @@ public String getPassword() { return password; } - public class DBMSConverter implements IStringConverter { - @Override - public DBMS convert(String value) { - return DBMS.valueOf(value); - } + public String getHost() { + return host; + } + + public int getPort() { + return port; } public boolean printProgressInformation() { return printProgressInformation; } + public boolean printProgressSummary() { + return printProgressSummary; + } + public int getTimeoutSeconds() { return timeoutSeconds; } + public int getMaxGeneratedDatabases() { + return maxGeneratedDatabases; + } + public int getErrorExitCode() { return errorExitCode; } @@ -114,4 +263,84 @@ public long getRandomSeed() { return randomSeed; } + public boolean testAggregateFunctionsPQS() { + return testAggregateFunctions; + } + + public boolean testOnlyWithMoreThanZeroRows() { + return testOnlyWithMoreThanZeroRows; + } + + public StringGenerationStrategy getRandomStringGenerationStrategy() { + return randomStringGenerationStrategy; + } + + public int getMaxStringConstantLength() { + return maxStringConstantLength; + } + + public boolean useConstantCaching() { + return useConstantCaching; + } + + public int getConstantCacheSize() { + return constantCacheSize; + } + + public boolean isHelp() { + return help; + } + + public boolean isDefaultPassword() { + return Objects.equals(password, DEFAULT_OPTIONS.password); + } + + public boolean isDefaultUsername() { + return Objects.equals(userName, DEFAULT_OPTIONS.userName); + } + + public String getDatabasePrefix() { + return databasePrefix; + } + + public boolean performConnectionTest() { + return useConnectionTest; + } + + public boolean serializeReproduceState() { + return serializeReproduceState; + } + + public boolean useReducer() { + return useReducer; + } + + public boolean reduceAST() { + return reduceAST; + } + + public long getMaxStatementReduceSteps() { + return maxStatementReduceSteps; + } + + public long getMaxStatementReduceTime() { + return maxStatementReduceTime; + } + + public long getMaxASTReduceSteps() { + return maxASTReduceSteps; + } + + public long getMaxASTReduceTime() { + return maxASTReduceTime; + } + + public boolean validateResultSizeOnly() { + return validateResultSizeOnly; + } + + public boolean canonicalizeSqlString() { + return canonicalizeSqlString; + } + } diff --git a/src/sqlancer/OracleFactory.java b/src/sqlancer/OracleFactory.java new file mode 100644 index 000000000..9d6e1704b --- /dev/null +++ b/src/sqlancer/OracleFactory.java @@ -0,0 +1,18 @@ +package sqlancer; + +import sqlancer.common.oracle.TestOracle; + +public interface OracleFactory> { + + TestOracle create(G globalState) throws Exception; + + /** + * Indicates whether the test oracle requires that all tables (including views) contain at least one row. + * + * @return whether the test oracle requires at least one row per table + */ + default boolean requiresAllTablesToContainRows() { + return false; + } + +} diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index fcc076a23..346567300 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -1,25 +1,39 @@ package sqlancer; -import java.io.FileWriter; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; -public abstract class ProviderAdapter, O> implements DatabaseProvider { +import sqlancer.StateToReproduce.OracleRunReproductionState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.schema.AbstractSchema; + +public abstract class ProviderAdapter, C>, O extends DBMSSpecificOptions>, C extends SQLancerDBConnection> + implements DatabaseProvider { private final Class globalClass; private final Class optionClass; - public ProviderAdapter(Class globalClass, Class optionClass) { + // Variables for QPG + Map queryPlanPool = new HashMap<>(); + static double[] weightedAverageReward; // static variable for sharing across all threads + int currentSelectRewards; + int currentSelectCounts; + int currentMutationOperator = -1; + + protected ProviderAdapter(Class globalClass, Class optionClass) { this.globalClass = globalClass; this.optionClass = optionClass; } - @Override - public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { - - } - @Override public StateToReproduce getStateToReproduce(String databaseName) { - return new StateToReproduce(databaseName); + return new StateToReproduce(databaseName, this); } @Override @@ -32,4 +46,213 @@ public Class getOptionClass() { return optionClass; } + @Override + public Reproducer generateAndTestDatabase(G globalState) throws Exception { + try { + generateDatabase(globalState); + checkViewsAreValid(globalState); + globalState.getManager().incrementCreateDatabase(); + + TestOracle oracle = getTestOracle(globalState); + for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { + try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { + assert localState != null; + try { + oracle.check(); + globalState.getManager().incrementSelectQueryCount(); + } catch (IgnoreMeException ignored) { + } catch (AssertionError e) { + Reproducer reproducer = oracle.getLastReproducer(); + if (reproducer != null) { + return reproducer; + } + throw e; + } + localState.executedWithoutError(); + } + } + } finally { + globalState.getConnection().close(); + } + return null; + } + + protected abstract void checkViewsAreValid(G globalState) throws SQLException; + + protected TestOracle getTestOracle(G globalState) throws Exception { + List> testOracleFactory = globalState.getDbmsSpecificOptions() + .getTestOracleFactory(); + boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() + .anyMatch(OracleFactory::requiresAllTablesToContainRows); + boolean userRequiresMoreThanZeroRows = globalState.getOptions().testOnlyWithMoreThanZeroRows(); + boolean checkZeroRows = testOracleRequiresMoreThanZeroRows || userRequiresMoreThanZeroRows; + if (checkZeroRows && globalState.getSchema().containsTableWithZeroRows(globalState)) { + if (globalState.getOptions().enableQPG()) { + addRowsToAllTables(globalState); + } else { + throw new IgnoreMeException(); + } + } + if (testOracleFactory.size() == 1) { + return testOracleFactory.get(0).create(globalState); + } else { + return new CompositeTestOracle<>(testOracleFactory.stream().map(o -> { + try { + return o.create(globalState); + } catch (Exception e1) { + throw new AssertionError(e1); + } + }).collect(Collectors.toList()), globalState); + } + } + + public abstract void generateDatabase(G globalState) throws Exception; + + // QPG: entry function + @Override + public void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws Exception { + if (weightedAverageReward == null) { + weightedAverageReward = initializeWeightedAverageReward(); // Same length as the list of mutators + } + try { + generateDatabase(globalState); + checkViewsAreValid(globalState); + globalState.getManager().incrementCreateDatabase(); + + Long executedQueryCount = 0L; + while (executedQueryCount < globalState.getOptions().getNrQueries()) { + int numOfNoNewQueryPlans = 0; + TestOracle oracle = getTestOracle(globalState); + while (executedQueryCount < globalState.getOptions().getNrQueries()) { + try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { + assert localState != null; + try { + oracle.check(); + String query = oracle.getLastQueryString(); + executedQueryCount += 1; + if (addQueryPlan(query, globalState)) { + numOfNoNewQueryPlans = 0; + } else { + numOfNoNewQueryPlans++; + } + globalState.getManager().incrementSelectQueryCount(); + } catch (IgnoreMeException e) { + + } + localState.executedWithoutError(); + } + // exit loop to mutate tables if no new query plans have been found after a while + if (numOfNoNewQueryPlans > globalState.getOptions().getQPGMaxMutationInterval()) { + mutateTables(globalState); + break; + } + } + } + } finally { + globalState.getConnection().close(); + } + } + + // QPG: mutate tables for a new database state + private synchronized boolean mutateTables(G globalState) throws Exception { + // Update rewards based on a set of newly generated queries in last iteration + if (currentMutationOperator != -1) { + weightedAverageReward[currentMutationOperator] += ((double) currentSelectRewards + / (double) currentSelectCounts) * globalState.getOptions().getQPGk(); + } + currentMutationOperator = -1; + + // Choose mutator based on the rewards + int selectedActionIndex = 0; + if (Randomly.getPercentage() < globalState.getOptions().getQPGProbability()) { + selectedActionIndex = globalState.getRandomly().getInteger(0, weightedAverageReward.length); + } else { + selectedActionIndex = DBMSCommon.getMaxIndexInDoubleArray(weightedAverageReward); + } + int reward = 0; + + try { + executeMutator(selectedActionIndex, globalState); + checkViewsAreValid(globalState); // Remove the invalid views + reward = checkQueryPlan(globalState); + } catch (IgnoreMeException | AssertionError e) { + } finally { + // Update rewards based on existing queries associated with the query plan pool + updateReward(selectedActionIndex, (double) reward / (double) queryPlanPool.size(), globalState); + currentMutationOperator = selectedActionIndex; + } + + // Clear the variables for storing the rewards of the action on a set of newly generated queries + currentSelectRewards = 0; + currentSelectCounts = 0; + return true; + } + + // QPG: add a query plan to the query plan pool and return true if the query plan is new + private boolean addQueryPlan(String selectStr, G globalState) throws Exception { + String queryPlan = getQueryPlan(selectStr, globalState); + + if (globalState.getOptions().logQueryPlan()) { + globalState.getLogger().writeQueryPlan(queryPlan); + } + + currentSelectCounts += 1; + if (queryPlanPool.containsKey(queryPlan)) { + return false; + } else { + queryPlanPool.put(queryPlan, selectStr); + currentSelectRewards += 1; + return true; + } + } + + // Obtain the reward of the current action based on the queries associated with the query plan pool + private int checkQueryPlan(G globalState) throws Exception { + int newQueryPlanFound = 0; + HashMap modifiedQueryPlan = new HashMap<>(); + for (Iterator> it = queryPlanPool.entrySet().iterator(); it.hasNext();) { + Map.Entry item = it.next(); + String queryPlan = item.getKey(); + String selectStr = item.getValue(); + String newQueryPlan = getQueryPlan(selectStr, globalState); + if (newQueryPlan.isEmpty()) { // Invalid query + it.remove(); + } else if (!queryPlan.equals(newQueryPlan)) { // A query plan has been changed + it.remove(); + modifiedQueryPlan.put(newQueryPlan, selectStr); + if (!queryPlanPool.containsKey(newQueryPlan)) { // A new query plan is found + newQueryPlanFound++; + } + } + } + queryPlanPool.putAll(modifiedQueryPlan); + return newQueryPlanFound; + } + + // QPG: update the reward of current action + private void updateReward(int actionIndex, double reward, G globalState) { + weightedAverageReward[actionIndex] += (reward - weightedAverageReward[actionIndex]) + * globalState.getOptions().getQPGk(); + } + + // QPG: initialize the weighted average reward of all mutation operators (required implementation in specific DBMS) + protected double[] initializeWeightedAverageReward() { + throw new UnsupportedOperationException(); + } + + // QPG: obtain the query plan of a query (required implementation in specific DBMS) + protected String getQueryPlan(String selectStr, G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + + // QPG: execute a mutation operator (required implementation in specific DBMS) + protected void executeMutator(int index, G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + + // QPG: add rows to all tables (required implementation in specific DBMS when enabling PQS oracle for QPG) + protected boolean addRowsToAllTables(G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + } diff --git a/src/sqlancer/Query.java b/src/sqlancer/Query.java deleted file mode 100644 index 622f48035..000000000 --- a/src/sqlancer/Query.java +++ /dev/null @@ -1,55 +0,0 @@ -package sqlancer; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; - -public abstract class Query { - - public abstract String getQueryString(); - - /** - * Whether the query could affect the schema (i.e., by add/deleting columns or tables). - * - * @return - */ - public abstract boolean couldAffectSchema(); - - /** - * - * @param con - * - * @return true if the query was successful, false otherwise - * - * @throws SQLException - */ - public abstract boolean execute(GlobalState globalState) throws SQLException; - - public abstract Collection getExpectedErrors(); - - @Override - public String toString() { - return getQueryString(); - } - - public ResultSet executeAndGet(GlobalState globalState) throws SQLException { - throw new AssertionError(); - } - - public boolean executeLogged(GlobalState globalState) throws SQLException { - logQueryString(globalState); - return execute(globalState); - } - - public ResultSet executeAndGetLogged(GlobalState globalState) throws SQLException { - logQueryString(globalState); - return executeAndGet(globalState); - } - - private void logQueryString(GlobalState globalState) { - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(getQueryString()); - } - } - -} diff --git a/src/sqlancer/QueryAdapter.java b/src/sqlancer/QueryAdapter.java deleted file mode 100644 index 0f8cb9ef2..000000000 --- a/src/sqlancer/QueryAdapter.java +++ /dev/null @@ -1,101 +0,0 @@ -package sqlancer; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collection; - -public class QueryAdapter extends Query { - - private final String query; - private final Collection expectedErrors; - private final boolean couldAffectSchema; - - public QueryAdapter(String query) { - this(query, new ArrayList<>()); - } - - public QueryAdapter(String query, boolean couldAffectSchema) { - this(query, new ArrayList<>(), couldAffectSchema); - } - - public QueryAdapter(String query, Collection expectedErrors) { - this.query = query; - this.expectedErrors = expectedErrors; - this.couldAffectSchema = false; - } - - public QueryAdapter(String query, Collection expectedErrors, boolean couldAffectSchema) { - this.query = query; - this.expectedErrors = expectedErrors; - this.couldAffectSchema = couldAffectSchema; - } - - @Override - public String getQueryString() { - return query; - } - - @Override - public boolean execute(GlobalState globalState) throws SQLException { - try (Statement s = globalState.getConnection().createStatement()) { - s.execute(query); - Main.nrSuccessfulActions.addAndGet(1); - return true; - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - checkException(e); - return false; - } - } - - public void checkException(Exception e) throws AssertionError { - boolean isExcluded = false; - for (String expectedError : expectedErrors) { - if (e.getMessage().contains(expectedError)) { - isExcluded = true; - break; - } - } - if (!isExcluded) { - throw new AssertionError(query, e); - } - } - - @Override - public ResultSet executeAndGet(GlobalState globalState) throws SQLException { - Statement s = globalState.getConnection().createStatement(); - ResultSet result = null; - try { - result = s.executeQuery(query); - Main.nrSuccessfulActions.addAndGet(1); - return result; - } catch (Exception e) { - s.close(); - boolean isExcluded = false; - Main.nrUnsuccessfulActions.addAndGet(1); - for (String expectedError : expectedErrors) { - if (e.getMessage().contains(expectedError)) { - isExcluded = true; - break; - } - } - if (!isExcluded) { - throw e; - } - } - return null; - } - - @Override - public boolean couldAffectSchema() { - return couldAffectSchema; - } - - @Override - public Collection getExpectedErrors() { - return expectedErrors; - } - -} diff --git a/src/sqlancer/QueryProvider.java b/src/sqlancer/QueryProvider.java deleted file mode 100644 index 99ee67d8c..000000000 --- a/src/sqlancer/QueryProvider.java +++ /dev/null @@ -1,8 +0,0 @@ -package sqlancer; - -import java.sql.SQLException; - -@FunctionalInterface -public interface QueryProvider { - Query getQuery(S globalState) throws SQLException; -} diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 1488ed43f..8494c189a 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -1,54 +1,73 @@ package sqlancer; import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Random; import java.util.function.Supplier; public final class Randomly { - private static final boolean USE_CACHING = true; - private static final int CACHE_SIZE = 100; + private static StringGenerationStrategy stringGenerationStrategy = StringGenerationStrategy.SOPHISTICATED; + private static int maxStringLength = 10; + private static boolean useCaching = true; + private static int cacheSize = 100; private final List cachedLongs = new ArrayList<>(); + private final List cachedIntegers = new ArrayList<>(); private final List cachedStrings = new ArrayList<>(); private final List cachedDoubles = new ArrayList<>(); private final List cachedBytes = new ArrayList<>(); - private static final String ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzöß!#<>/.,~-+'*()[]{} ^*?%_\t\n\r|&\\"; private Supplier provider; private static final ThreadLocal THREAD_RANDOM = new ThreadLocal<>(); + private long seed; private void addToCache(long val) { - if (USE_CACHING && cachedLongs.size() < CACHE_SIZE && !cachedLongs.contains(val)) { + if (useCaching && cachedLongs.size() < cacheSize && !cachedLongs.contains(val)) { cachedLongs.add(val); } } + private void addToCache(int val) { + if (useCaching && cachedIntegers.size() < cacheSize && !cachedIntegers.contains(val)) { + cachedIntegers.add(val); + } + } + private void addToCache(double val) { - if (USE_CACHING && cachedDoubles.size() < CACHE_SIZE && !cachedDoubles.contains(val)) { + if (useCaching && cachedDoubles.size() < cacheSize && !cachedDoubles.contains(val)) { cachedDoubles.add(val); } } private void addToCache(String val) { - if (USE_CACHING && cachedStrings.size() < CACHE_SIZE && !cachedStrings.contains(val)) { + if (useCaching && cachedStrings.size() < cacheSize && !cachedStrings.contains(val)) { cachedStrings.add(val); } } private Long getFromLongCache() { - if (!USE_CACHING || cachedLongs.isEmpty()) { + if (!useCaching || cachedLongs.isEmpty()) { return null; } else { return Randomly.fromList(cachedLongs); } } + private Integer getFromIntegerCache() { + if (!useCaching || cachedIntegers.isEmpty()) { + return null; + } else { + return Randomly.fromList(cachedIntegers); + } + } + private Double getFromDoubleCache() { - if (!USE_CACHING) { + if (!useCaching) { return null; } if (Randomly.getBoolean() && !cachedLongs.isEmpty()) { @@ -61,33 +80,22 @@ private Double getFromDoubleCache() { } private String getFromStringCache() { - if (!USE_CACHING) { + if (!useCaching) { return null; } if (Randomly.getBoolean() && !cachedLongs.isEmpty()) { return String.valueOf(Randomly.fromList(cachedLongs)); } else if (Randomly.getBoolean() && !cachedDoubles.isEmpty()) { return String.valueOf(Randomly.fromList(cachedDoubles)); - } else if (Randomly.getBoolean() && !cachedBytes.isEmpty()) { + } else if (Randomly.getBoolean() && !cachedBytes.isEmpty() + && stringGenerationStrategy == StringGenerationStrategy.SOPHISTICATED) { return new String(Randomly.fromList(cachedBytes)); } else if (!cachedStrings.isEmpty()) { String randomString = Randomly.fromList(cachedStrings); if (Randomly.getBoolean()) { return randomString; } else { - if (Randomly.getBoolean()) { - return randomString.toLowerCase(); - } else if (Randomly.getBoolean()) { - return randomString.toUpperCase(); - } else { - char[] chars = randomString.toCharArray(); - if (chars.length != 0) { - for (int i = 0; i < Randomly.smallNumber(); i++) { - chars[getInteger(0, chars.length)] = ALPHABET.charAt(getInteger(0, ALPHABET.length())); - } - } - return new String(chars); - } + return stringGenerationStrategy.transformCachedString(this, randomString); } } else { return null; @@ -95,7 +103,7 @@ private String getFromStringCache() { } private static boolean cacheProbability() { - return USE_CACHING && getNextLong(0, 3) == 1; + return useCaching && getNextLong(0, 3) == 1; } // CACHING END @@ -127,6 +135,12 @@ public static List nonEmptySubset(List columns, int nr) { return extractNrRandomColumns(columns, nr); } + public static List nonEmptySubsetLeast(List columns, int min) { + int nr = getNextInt(min, columns.size() + 1); + assert nr <= columns.size(); + return extractNrRandomColumns(columns, nr); + } + public static List nonEmptySubsetPotentialDuplicates(List columns) { List arr = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { @@ -142,17 +156,12 @@ public static List subset(List columns) { public static List subset(int nr, @SuppressWarnings("unchecked") T... values) { List list = new ArrayList<>(); - for (T val : values) { - list.add(val); - } + Collections.addAll(list, values); return extractNrRandomColumns(list, nr); } public static List subset(@SuppressWarnings("unchecked") T... values) { - List list = new ArrayList<>(); - for (T val : values) { - list.add(val); - } + List list = new ArrayList<>(Arrays.asList(values)); return subset(list); } @@ -168,13 +177,17 @@ public static List extractNrRandomColumns(List columns, int nr) { public static int smallNumber() { // no need to cache for small numbers - return (int) (Math.abs(getThreadRandom().get().nextGaussian()) * 2); + return (int) (Math.abs(getThreadRandom().get().nextGaussian())) * 2; } public static boolean getBoolean() { return getThreadRandom().get().nextBoolean(); } + public static double getPercentage() { + return getThreadRandom().get().nextDouble(); + } + private static ThreadLocal getThreadRandom() { if (THREAD_RANDOM.get() == null) { // a static method has been called, before Randomly was instantiated @@ -199,59 +212,137 @@ public long getInteger() { } } - public String getString() { - if (smallBiasProbability()) { - return Randomly.fromOptions("TRUE", "FALSE", "0.0", "-0.0", "1e500", "-1e500"); - } - if (cacheProbability()) { - String s = getFromStringCache(); - if (s != null) { - return s; + public enum StringGenerationStrategy { + + NUMERIC { + @Override + public String getString(Randomly r) { + return getStringOfAlphabet(r, NUMERIC_ALPHABET); } - } - int n = ALPHABET.length(); + }, + ALPHANUMERIC { + @Override + public String getString(Randomly r) { + return getStringOfAlphabet(r, ALPHANUMERIC_ALPHABET); - StringBuilder sb = new StringBuilder(); + } - int chars; - if (Randomly.getBoolean()) { - chars = Randomly.smallNumber(); - } else { - chars = getInteger(0, 30); - } - for (int i = 0; i < chars; i++) { - if (Randomly.getBooleanWithRatherLowProbability()) { - char val = (char) getInteger(); - if (val != 0) { - sb.append(val); + }, + ALPHANUMERIC_SPECIALCHAR { + @Override + public String getString(Randomly r) { + return getStringOfAlphabet(r, ALPHANUMERIC_SPECIALCHAR_ALPHABET); + + } + + }, + SOPHISTICATED { + + private static final String ALPHABET = ALPHANUMERIC_SPECIALCHAR_ALPHABET; + + @Override + public String getString(Randomly r) { + if (smallBiasProbability()) { + return Randomly.fromOptions("TRUE", "FALSE", "0.0", "-0.0", "1e500", "-1e500"); } - } else { - sb.append(ALPHABET.charAt(getNextInt(0, n))); + if (cacheProbability()) { + String s = r.getFromStringCache(); + if (s != null) { + return s; + } + } + + int n = ALPHABET.length(); + + StringBuilder sb = new StringBuilder(); + + int chars = getStringLength(r); + for (int i = 0; i < chars; i++) { + if (Randomly.getBooleanWithRatherLowProbability()) { + char val = (char) r.getInteger(); + if (val != 0) { + sb.append(val); + } + } else { + sb.append(ALPHABET.charAt(getNextInt(0, n))); + } + } + while (Randomly.getBooleanWithSmallProbability()) { + String[][] pairs = { { "{", "}" }, { "[", "]" }, { "(", ")" } }; + int idx = (int) Randomly.getNotCachedInteger(0, pairs.length); + int left = (int) Randomly.getNotCachedInteger(0, sb.length() + 1); + sb.insert(left, pairs[idx][0]); + int right = (int) Randomly.getNotCachedInteger(left + 1, sb.length() + 1); + sb.insert(right, pairs[idx][1]); + } + if (r.provider != null) { + while (Randomly.getBooleanWithSmallProbability()) { + if (sb.length() == 0) { + sb.append(r.provider.get()); + } else { + sb.insert((int) Randomly.getNotCachedInteger(0, sb.length()), r.provider.get()); + } + } + } + + String s = sb.toString(); + + r.addToCache(s); + return s; } - } - while (Randomly.getBooleanWithSmallProbability()) { - String[][] pairs = { { "{", "}" }, { "[", "]" }, { "(", ")" } }; - int idx = (int) Randomly.getNotCachedInteger(0, pairs.length); - int left = (int) Randomly.getNotCachedInteger(0, sb.length() + 1); - sb.insert(left, pairs[idx][0]); - int right = (int) Randomly.getNotCachedInteger(left + 1, sb.length() + 1); - sb.insert(right, pairs[idx][1]); - } - if (provider != null) { - while (Randomly.getBooleanWithSmallProbability()) { - if (sb.length() == 0) { - sb.append(provider.get()); + + public String transformCachedString(Randomly r, String randomString) { + if (Randomly.getBoolean()) { + return randomString.toLowerCase(); + } else if (Randomly.getBoolean()) { + return randomString.toUpperCase(); } else { - sb.insert((int) Randomly.getNotCachedInteger(0, sb.length()), provider.get()); + char[] chars = randomString.toCharArray(); + if (chars.length != 0) { + for (int i = 0; i < Randomly.smallNumber(); i++) { + chars[r.getInteger(0, chars.length)] = ALPHABET.charAt(r.getInteger(0, ALPHABET.length())); + } + } + return new String(chars); } } + + }; + + private static final String ALPHANUMERIC_SPECIALCHAR_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#<>/.,~-+'*()[]{} ^*?%_\t\n\r|&\\"; + private static final String ALPHANUMERIC_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + private static final String NUMERIC_ALPHABET = "0123456789"; + + private static int getStringLength(Randomly r) { + int chars; + if (Randomly.getBoolean()) { + chars = Randomly.smallNumber(); + } else { + chars = r.getInteger(0, maxStringLength); + } + return chars; + } + + private static String getStringOfAlphabet(Randomly r, String alphabet) { + int chars = getStringLength(r); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < chars; i++) { + sb.append(alphabet.charAt(getNextInt(0, alphabet.length()))); + } + return sb.toString(); } - String s = sb.toString(); + public abstract String getString(Randomly r); + + public String transformCachedString(Randomly r, String s) { + return s; + } - addToCache(s); - return s; + } + + public String getString() { + return stringGenerationStrategy.getString(this); } public byte[] getBytes() { @@ -275,7 +366,6 @@ public long getNonZeroInteger() { do { value = getInteger(); } while (value == 0); - assert value != 0; addToCache(value); return value; } @@ -298,6 +388,24 @@ public long getPositiveInteger() { return value; } + public int getPositiveIntegerInt() { + if (cacheProbability()) { + Integer value = getFromIntegerCache(); + if (value != null && value >= 0) { + return value; + } + } + int value; + if (smallBiasProbability()) { + value = Randomly.fromOptions(0, Integer.MAX_VALUE, 1); + } else { + value = getNextInt(0, Integer.MAX_VALUE); + } + addToCache(value); + assert value >= 0; + return value; + } + public double getFiniteDouble() { while (true) { double val = getDouble(); @@ -349,8 +457,19 @@ public long getLong(long left, long right) { return getNextLong(left, right); } + public BigInteger getBigInteger(BigInteger left, BigInteger right) { + if (left.equals(right)) { + return left; + } + BigInteger result = new BigInteger(String.valueOf(getInteger(left.intValue(), right.intValue()))); + if (result.compareTo(left) < 0 && result.compareTo(right) > 0) { + throw new IgnoreMeException(); + } + return result; + } + public BigDecimal getRandomBigDecimal() { - return new BigDecimal(getThreadRandom().get().nextDouble()); + return BigDecimal.valueOf(getThreadRandom().get().nextDouble()); } public long getPositiveIntegerNotNull() { @@ -366,10 +485,6 @@ public static long getNonCachedInteger() { return getThreadRandom().get().nextLong(); } - public static long getPositiveNonCachedInteger() { - return getNextLong(1, Long.MAX_VALUE); - } - public static long getPositiveOrZeroNonCachedInteger() { return getNextLong(0, Long.MAX_VALUE); } @@ -383,11 +498,12 @@ public Randomly(Supplier provider) { } public Randomly() { - getThreadRandom().set(new Random()); + THREAD_RANDOM.set(new Random()); } public Randomly(long seed) { - getThreadRandom().set(new Random(seed)); + this.seed = seed; + THREAD_RANDOM.set(new Random(seed)); } public static double getUncachedDouble() { @@ -403,6 +519,15 @@ public String getChar() { } } + public String getAlphabeticChar() { + while (true) { + String s = getChar(); + if (Character.isAlphabetic(s.charAt(0))) { + return s; + } + } + } + // see https://stackoverflow.com/a/2546158 // uniformity does not seem to be important for us // SQLancer previously used ThreadLocalRandom.current().nextLong(lower, upper) @@ -413,11 +538,22 @@ private static long getNextLong(long lower, long upper) { if (lower == upper) { return lower; } - return (long) (getThreadRandom().get().longs(lower, upper).findFirst().getAsLong()); + return getThreadRandom().get().longs(lower, upper).findFirst().getAsLong(); } private static int getNextInt(int lower, int upper) { return (int) getNextLong(lower, upper); } + public long getSeed() { + return seed; + } + + public static void initialize(MainOptions options) { + stringGenerationStrategy = options.getRandomStringGenerationStrategy(); + maxStringLength = options.getMaxStringConstantLength(); + useCaching = options.useConstantCaching(); + cacheSize = options.getConstantCacheSize(); + } + } diff --git a/src/sqlancer/Reducer.java b/src/sqlancer/Reducer.java new file mode 100644 index 000000000..0e6589262 --- /dev/null +++ b/src/sqlancer/Reducer.java @@ -0,0 +1,7 @@ +package sqlancer; + +public interface Reducer> { + + void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception; + +} diff --git a/src/sqlancer/Reproducer.java b/src/sqlancer/Reproducer.java new file mode 100644 index 000000000..460cc810d --- /dev/null +++ b/src/sqlancer/Reproducer.java @@ -0,0 +1,15 @@ +package sqlancer; + +public interface Reproducer> { + boolean bugStillTriggers(G globalState); + + /** + * Describes how to trigger the bug on the database set up by the reduced statements (e.g., the oracle queries to + * run and the failure to expect), so that the reduced test case is complete without the reproducer object. + * + * @return a human-readable description, or null if the reproducer does not provide one + */ + default String getBugInformation() { + return null; + } +} diff --git a/src/sqlancer/SQLConnection.java b/src/sqlancer/SQLConnection.java new file mode 100644 index 000000000..ae56c781f --- /dev/null +++ b/src/sqlancer/SQLConnection.java @@ -0,0 +1,34 @@ +package sqlancer; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.sql.Statement; + +public class SQLConnection implements SQLancerDBConnection { + + private final Connection connection; + + public SQLConnection(Connection connection) { + this.connection = connection; + } + + @Override + public String getDatabaseVersion() throws SQLException { + DatabaseMetaData meta = connection.getMetaData(); + return meta.getDatabaseProductVersion(); + } + + @Override + public void close() throws SQLException { + connection.close(); + } + + public Statement prepareStatement(String arg) throws SQLException { + return connection.prepareStatement(arg); + } + + public Statement createStatement() throws SQLException { + return connection.createStatement(); + } +} diff --git a/src/sqlancer/SQLGlobalState.java b/src/sqlancer/SQLGlobalState.java new file mode 100644 index 000000000..534086472 --- /dev/null +++ b/src/sqlancer/SQLGlobalState.java @@ -0,0 +1,30 @@ +package sqlancer; + +import sqlancer.common.query.Query; +import sqlancer.common.schema.AbstractSchema; + +/** + * Represents a global state that is valid for a testing session on a given database. + * + * @param + * the option parameter + * @param + * the schema parameter + */ +public abstract class SQLGlobalState, S extends AbstractSchema> + extends GlobalState { + + @Override + protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { + boolean logExecutionTime = getOptions().logExecutionTime(); + if (success && getOptions().printSucceedingStatements()) { + System.out.println(q.getQueryString()); + } + if (logExecutionTime) { + getLogger().writeCurrent(" -- " + timer.end().asString()); + } + if (q.couldAffectSchema()) { + updateSchema(); + } + } +} diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java new file mode 100644 index 000000000..efb4fab67 --- /dev/null +++ b/src/sqlancer/SQLProviderAdapter.java @@ -0,0 +1,44 @@ +package sqlancer; + +import java.util.List; + +import sqlancer.common.log.LoggableFactory; +import sqlancer.common.log.SQLLoggableFactory; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; + +public abstract class SQLProviderAdapter>, O extends DBMSSpecificOptions>> + extends ProviderAdapter { + protected SQLProviderAdapter(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + @Override + public LoggableFactory getLoggableFactory() { + return new SQLLoggableFactory(); + } + + @Override + protected void checkViewsAreValid(G globalState) { + List> views = globalState.getSchema().getViews(); + for (AbstractTable view : views) { + SQLQueryAdapter q = new SQLQueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); + try { + if (!q.execute(globalState)) { + dropView(globalState, view.getName()); + } + } catch (Throwable t) { + dropView(globalState, view.getName()); + } + } + } + + private void dropView(G globalState, String viewName) { + try { + globalState.executeStatement(new SQLQueryAdapter("DROP VIEW " + viewName, true)); + } catch (Throwable t2) { + throw new IgnoreMeException(); + } + } +} diff --git a/src/sqlancer/SQLancerDBConnection.java b/src/sqlancer/SQLancerDBConnection.java new file mode 100644 index 000000000..1724dda6c --- /dev/null +++ b/src/sqlancer/SQLancerDBConnection.java @@ -0,0 +1,6 @@ +package sqlancer; + +public interface SQLancerDBConnection extends AutoCloseable { + + String getDatabaseVersion() throws Exception; +} diff --git a/src/sqlancer/StandaloneReducer.java b/src/sqlancer/StandaloneReducer.java new file mode 100644 index 000000000..813160060 --- /dev/null +++ b/src/sqlancer/StandaloneReducer.java @@ -0,0 +1,140 @@ +package sqlancer; + +import java.io.FileWriter; +import java.io.PrintWriter; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.Query; + +/** + * A standalone tool to reduce bug-triggering SQL statements using the delta debugging algorithm. + */ +public class StandaloneReducer { + private int partitionNum = 2; + private final StateToReproduce originalState; + private final DatabaseProvider databaseProvider; + private final Path outputPath; + + public StandaloneReducer(Path inputPath, Path outputPath) throws Exception { + this.originalState = StateToReproduce.deserialize(inputPath); + this.databaseProvider = originalState.getDatabaseProvider(); + if (this.databaseProvider == null) { + throw new IllegalStateException("Failed to get database provider from .ser file"); + } + this.outputPath = outputPath != null ? outputPath + : Paths.get(inputPath.toString().replaceAll("\\.ser$", ".sql")); + } + + /** + * Performs the main reduction algorithm using partition-based delta debugging. + * + * @return List of reduced SQL statements that still trigger bugs. + */ + public List> reduce() throws Exception { + List> queries = new ArrayList<>(originalState.getStatements()); + if (queries.size() <= 1) { + return queries; + } + + partitionNum = 2; + while (queries.size() >= 2) { + boolean changedInThisPass = false; + List> result = tryReduction(queries); + + if (result.size() < queries.size()) { + queries = result; + changedInThisPass = true; + } + + if (changedInThisPass) { + partitionNum = 2; + } else { + if (partitionNum >= queries.size()) { + break; + } + partitionNum = Math.min(partitionNum * 2, queries.size()); + } + } + + try (PrintWriter writer = new PrintWriter(new FileWriter(outputPath.toFile()))) { + for (Query query : queries) { + writer.println(query.getQueryString()); + } + } + System.out.println("Reduction completed successfully! SQL statements written to: " + outputPath.toString()); + System.out.println("Final size: " + queries.size() + " statements (" + + String.format("%.1f", (1.0 - (double) queries.size() / originalState.getStatements().size()) * 100) + + "% reduction)"); + + return queries; + } + + private List> tryReduction(List> queries) throws Exception { + int start = 0; + int subLength = queries.size() / partitionNum; + + while (start < queries.size()) { + List> candidateQueries = new ArrayList<>(queries); + int endPoint = Math.min(start + subLength, candidateQueries.size()); + candidateQueries.subList(start, endPoint).clear(); + + if (testExceptionStillExists(candidateQueries)) { + return candidateQueries; + } + + start += subLength; + } + + return queries; + } + + // Test if bug still exists with reduced query set + @SuppressWarnings("unchecked") + private , O extends DBMSSpecificOptions, C extends SQLancerDBConnection> boolean testExceptionStillExists( + List> queries) { + try { + DatabaseProvider typedProvider = (DatabaseProvider) databaseProvider; + G globalState = typedProvider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + + try (C connection = typedProvider.createDatabase(globalState)) { + globalState.setConnection(connection); + for (Query query : queries) { + try { + Query typedQuery = (Query) query; + typedQuery.execute(globalState); + } catch (Throwable e) { + // Any exception not declared as an expected error by the query indicates that an (unexpected) + // exception still exists + return true; + } + } + // No exception occurred + return false; + } + } catch (Throwable e) { + return true; + } + } + + public static void main(String[] args) { + try { + if (args.length == 0) { + System.err.println( + "Usage: java -cp target/sqlancer-2.0.0.jar sqlancer.StandaloneReducer [output-file]"); + System.exit(1); + } + Path inputPath = Paths.get(args[0]); + Path outputPath = args.length > 1 ? Paths.get(args[1]) : null; + + StandaloneReducer reducer = new StandaloneReducer(inputPath, outputPath); + reducer.reduce(); + } catch (Throwable e) { + System.err.println("ERROR: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } +} diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index ea52d7bbb..17bb367fd 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -1,43 +1,38 @@ package sqlancer; +import java.io.Closeable; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Map; - -import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ast.ClickHouseConstant; -import sqlancer.clickhouse.ast.ClickHouseExpression; -import sqlancer.mysql.MySQLSchema.MySQLColumn; -import sqlancer.mysql.ast.MySQLConstant; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.ast.PostgresConstant; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.sqlite3.ast.SQLite3Constant; -import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; - -public class StateToReproduce { - - public final List statements = new ArrayList<>(); - public String queryString; + +import sqlancer.common.query.Query; + +public class StateToReproduce implements Serializable { + private static final long serialVersionUID = 1L; + + private List> statements = new ArrayList<>(); private final String databaseName; + private transient DatabaseProvider databaseProvider; + public String databaseVersion; protected long seedValue; - public String values; - String exception; - public String queryTargetedTablesString; + public transient OracleRunReproductionState localState; - public String queryTargetedColumnsString; - - public StateToReproduce(String databaseName) { + public StateToReproduce(String databaseName, DatabaseProvider databaseProvider) { this.databaseName = databaseName; + this.databaseProvider = databaseProvider; } public String getException() { @@ -52,101 +47,146 @@ public String getDatabaseVersion() { return databaseVersion; } - public List getStatements() { - return statements; + public DatabaseProvider getDatabaseProvider() { + return databaseProvider; } - public String getQueryString() { - return queryString; + /** + * Logs the statement string without executing the corresponding statement. + * + * @param queryString + * the query string to be logged + */ + public void logStatement(String queryString) { + if (queryString == null) { + throw new IllegalArgumentException(); + } + logStatement(databaseProvider.getLoggableFactory().getQueryForStateToReproduce(queryString)); + } + + /** + * Logs the statement without executing it. + * + * @param query + * the query to be logged + */ + public void logStatement(Query query) { + if (query == null) { + throw new IllegalArgumentException(); + } + statements.add(query); + } + + public List> getStatements() { + return Collections.unmodifiableList(statements); + } + + /** + * @deprecated + */ + @Deprecated + public void commentStatements() { + for (int i = 0; i < statements.size(); i++) { + Query statement = statements.get(i); + Query newQuery = databaseProvider.getLoggableFactory().commentOutQuery(statement); + statements.set(i, newQuery); + } } public long getSeedValue() { return seedValue; } - public static class MySQLStateToReproduce extends StateToReproduce { + /** + * Returns a local state in which a test oracle can save useful information about a single run. If the local state + * is closed without indicating access to it, the local statements will be added to the global state. + * + * @return the local state for logging + */ + public OracleRunReproductionState getLocalState() { + return localState; + } - public Map randomRowValues; + /** + * State information that is logged if the test oracle finds a bug or if an exception is thrown. + */ + public class OracleRunReproductionState implements Closeable { - public MySQLExpression whereClause; + private final List> statements = new ArrayList<>(); - public String queryThatSelectsRow; + private boolean success; - public MySQLStateToReproduce(String databaseName) { - super(databaseName); + public OracleRunReproductionState() { + StateToReproduce.this.localState = this; } - public Map getRandomRowValues() { - return randomRowValues; + public void executedWithoutError() { + this.success = true; } - public MySQLExpression getWhereClause() { - return whereClause; + public void log(String s) { + statements.add(databaseProvider.getLoggableFactory().getQueryForStateToReproduce(s)); } - } - - public static class SQLite3StateToReproduce extends StateToReproduce { - public Map randomRowValues; - - public SQLite3Expression whereClause; - - public SQLite3StateToReproduce(String databaseName) { - super(databaseName); + public List> getStatements() { + return Collections.unmodifiableList(statements); } - public Map getRandomRowValues() { - return randomRowValues; - } + @Override + public void close() { + if (!success) { + StateToReproduce.this.statements.addAll(statements); + } - public SQLite3Expression getWhereClause() { - return whereClause; } } - public static class PostgresStateToReproduce extends StateToReproduce { - - public Map randomRowValues; - - public PostgresExpression whereClause; - - public String queryThatSelectsRow; - - public PostgresStateToReproduce(String databaseName) { - super(databaseName); - } + public OracleRunReproductionState createLocalState() { + return new OracleRunReproductionState(); + } - public Map getRandomRowValues() { - return randomRowValues; + public void serialize(Path path) { + try (ObjectOutputStream oos = new ObjectOutputStream(Files.newOutputStream(path))) { + oos.writeObject(this); + } catch (IOException e) { + throw new AssertionError(e); } + } - public PostgresExpression getWhereClause() { - return whereClause; + public static StateToReproduce deserialize(Path path) { + try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(path))) { + return (StateToReproduce) ois.readObject(); + } catch (IOException | ClassNotFoundException e) { + throw new AssertionError(e); } - } - public static class ClickHouseStateToReproduce extends StateToReproduce { - - public Map randomRowValues; - - public ClickHouseExpression whereClause; + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); - public String queryThatSelectsRow; - - public ClickHouseStateToReproduce(String databaseName) { - super(databaseName); - } - - public Map getRandomRowValues() { - return randomRowValues; - } + out.writeObject(this.databaseProvider != null ? this.databaseProvider.getDBMSName() : null); + } - public ClickHouseExpression getWhereClause() { - return whereClause; + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + String dbmsName = (String) in.readObject(); + + DatabaseProvider provider = null; + if (dbmsName != null) { + List> providers = Main.getDBMSProviders(); + for (DatabaseProvider p : providers) { + if (p.getDBMSName().equals(dbmsName)) { + provider = p; + break; + } + } } + this.databaseProvider = provider; + } + public void setStatements(List> statements) { + this.statements = statements; } } diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index caf7c5201..4f8f48b8f 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -1,10 +1,11 @@ package sqlancer; -import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -public class StatementExecutor, A extends AbstractAction> { +import sqlancer.common.query.Query; + +public class StatementExecutor, A extends AbstractAction> { private final G globalState; private final A[] actions; @@ -13,7 +14,7 @@ public class StatementExecutor, A extends AbstractActio @FunctionalInterface public interface AfterQueryAction { - void notify(Query q) throws SQLException; + void notify(Query q) throws Exception; } @FunctionalInterface @@ -28,7 +29,8 @@ public StatementExecutor(G globalState, A[] actions, ActionMapper mapping, this.queryConsumer = queryConsumer; } - public void executeStatements() throws SQLException { + @SuppressWarnings("unchecked") + public void executeStatements() throws Exception { Randomly r = globalState.getRandomly(); int[] nrRemaining = new int[actions.length]; List availableActions = new ArrayList<>(); @@ -58,21 +60,21 @@ public void executeStatements() throws SQLException { assert nextAction != null; assert nrRemaining[i] > 0; nrRemaining[i]--; + @SuppressWarnings("rawtypes") Query query = null; try { boolean success; int nrTries = 0; do { query = nextAction.getQuery(globalState); - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(query.getQueryString()); - } - success = globalState.getManager().execute(query); - } while (!success && nrTries++ < globalState.getOptions().getNrStatementRetryCount()); - } catch (IgnoreMeException e) { + success = globalState.executeStatement(query); + } while (nextAction.canBeRetried() && !success + && nrTries++ < globalState.getOptions().getNrStatementRetryCount()); + } catch (IgnoreMeException ignored) { } if (query != null && query.couldAffectSchema()) { + globalState.updateSchema(); queryConsumer.notify(query); } total--; diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java new file mode 100644 index 000000000..6545fb2af --- /dev/null +++ b/src/sqlancer/StatementReducer.java @@ -0,0 +1,147 @@ +package sqlancer; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.Query; + +public class StatementReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> + implements Reducer { + private final DatabaseProvider provider; + private boolean observedChange; + private int partitionNum; + + private long currentReduceSteps; + private long currentReduceTime; + + private long maxReduceSteps; + private long maxReduceTime; + + Instant timeOfReductionBegins; + + public StatementReducer(DatabaseProvider provider) { + this.provider = provider; + } + + private boolean hasNotReachedLimit(long curr, long limit) { + if (limit == MainOptions.NO_REDUCE_LIMIT) { + return true; + } + return curr < limit; + } + + @SuppressWarnings("unchecked") + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + + maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); + + List> knownToReproduceBugStatements = new ArrayList<>(); + for (Query stat : state.getState().getStatements()) { + knownToReproduceBugStatements.add((Query) stat); + } + + // System.out.println("Starting query:"); + // Main.StateLogger logger = newGlobalState.getLogger(); + // printQueries(knownToReproduceBugStatements); + // System.out.println(); + + if (knownToReproduceBugStatements.size() <= 1) { + return; + } + + timeOfReductionBegins = Instant.now(); + currentReduceSteps = 0; + currentReduceTime = 0; + partitionNum = 2; + + while (knownToReproduceBugStatements.size() >= 2 && hasNotReachedLimit(currentReduceSteps, maxReduceSteps) + && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { + observedChange = false; + + knownToReproduceBugStatements = tryReduction(state, reproducer, newGlobalState, + knownToReproduceBugStatements); + + if (!observedChange) { + if (partitionNum == knownToReproduceBugStatements.size()) { + break; + } + // increase the search granularity + partitionNum = Math.min(partitionNum * 2, knownToReproduceBugStatements.size()); + } + } + + // System.out.println("Reduced query:"); + // printQueries(knownToReproduceBugStatements); + newGlobalState.getState().setStatements(new ArrayList<>(knownToReproduceBugStatements)); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "Statement reduction finished; the following statements remain"); + + } + + private List> tryReduction(G state, // NOPMD + Reproducer reproducer, G newGlobalState, List> knownToReproduceBugStatements) throws Exception { + + List> statements = knownToReproduceBugStatements; + + int start = 0; + int subLength = statements.size() / partitionNum; + while (start < statements.size()) { + // newStatements = candidate[:start] + candidate[start+subLength:] + // in other word, remove [start, start+subLength) from candidates + try (C con2 = provider.createDatabase(newGlobalState)) { + newGlobalState.setConnection(con2); + List> candidateStatements = new ArrayList<>(statements); + int endPoint = Math.min(start + subLength, candidateStatements.size()); + candidateStatements.subList(start, endPoint).clear(); + newGlobalState.getState().setStatements(new ArrayList<>(candidateStatements)); + + for (Query s : candidateStatements) { + try { + s.execute(newGlobalState); + } catch (Throwable ignoredException) { + // ignore + } + } + try { + if (reproducer.bugStillTriggers(newGlobalState)) { + observedChange = true; + statements = candidateStatements; + partitionNum = Math.max(partitionNum - 1, 2); + // reproducer.outputHook((SQLite3GlobalState) newGlobalState); + newGlobalState.getLogger().logReduced(newGlobalState.getState()); + break; + + } + } catch (Throwable ignoredException) { + + } + } catch (Exception e) { + e.printStackTrace(); + } + + currentReduceSteps++; + Instant currentInstant = Instant.now(); + + currentReduceTime = Duration.between(timeOfReductionBegins, currentInstant).getSeconds(); + if (!hasNotReachedLimit(currentReduceSteps, maxReduceSteps) + || !hasNotReachedLimit(currentReduceTime, maxReduceTime)) { + return statements; + } + start = start + subLength; + } + return statements; + } + + @SuppressWarnings("unused") + private void printQueries(List> statements) { + System.out.println("==============================="); + for (Query q : statements) { + System.out.println(q.getLogString()); + } + System.out.println("==============================="); + } +} diff --git a/src/sqlancer/TestOracle.java b/src/sqlancer/TestOracle.java deleted file mode 100644 index 40586a0f7..000000000 --- a/src/sqlancer/TestOracle.java +++ /dev/null @@ -1,13 +0,0 @@ -package sqlancer; - -import java.sql.SQLException; - -public interface TestOracle { - - void check() throws SQLException; - - default boolean onlyWorksForNonEmptyTables() { - return false; - } - -} diff --git a/src/sqlancer/ast/newast/NewAliasNode.java b/src/sqlancer/ast/newast/NewAliasNode.java deleted file mode 100644 index 924533214..000000000 --- a/src/sqlancer/ast/newast/NewAliasNode.java +++ /dev/null @@ -1,21 +0,0 @@ -package sqlancer.ast.newast; - -public class NewAliasNode implements Node { - - private final Node expr; - private final String alias; - - public NewAliasNode(Node expr, String alias) { - this.expr = expr; - this.alias = alias; - } - - public Node getExpr() { - return expr; - } - - public String getAlias() { - return alias; - } - -} diff --git a/src/sqlancer/ast/newast/NewBetweenOperatorNode.java b/src/sqlancer/ast/newast/NewBetweenOperatorNode.java deleted file mode 100644 index 91c54a5f1..000000000 --- a/src/sqlancer/ast/newast/NewBetweenOperatorNode.java +++ /dev/null @@ -1,33 +0,0 @@ -package sqlancer.ast.newast; - -public class NewBetweenOperatorNode implements Node { - - protected Node left; - protected Node middle; - protected Node right; - protected boolean isTrue; - - public NewBetweenOperatorNode(Node left, Node middle, Node right, boolean isTrue) { - this.left = left; - this.middle = middle; - this.right = right; - this.isTrue = isTrue; - } - - public Node getLeft() { - return left; - } - - public Node getMiddle() { - return middle; - } - - public Node getRight() { - return right; - } - - public boolean isTrue() { - return isTrue; - } - -} diff --git a/src/sqlancer/ast/newast/NewBinaryOperatorNode.java b/src/sqlancer/ast/newast/NewBinaryOperatorNode.java deleted file mode 100644 index 3e703e8bd..000000000 --- a/src/sqlancer/ast/newast/NewBinaryOperatorNode.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.ast.newast; - -import sqlancer.ast.BinaryOperatorNode.Operator; - -public class NewBinaryOperatorNode implements Node { - - protected final Operator op; - protected final Node left; - protected final Node right; - - public NewBinaryOperatorNode(Node left, Node right, Operator op) { - this.left = left; - this.right = right; - this.op = op; - } - - public String getOperatorRepresentation() { - return op.getTextRepresentation(); - } - - public Node getLeft() { - return left; - } - - public Node getRight() { - return right; - } - -} diff --git a/src/sqlancer/ast/newast/NewCaseOperatorNode.java b/src/sqlancer/ast/newast/NewCaseOperatorNode.java deleted file mode 100644 index 3c0173536..000000000 --- a/src/sqlancer/ast/newast/NewCaseOperatorNode.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.ast.newast; - -import java.util.List; - -public class NewCaseOperatorNode implements Node { - - private final List> conditions; - private final List> expressions; - private final Node elseExpr; - private final Node switchCondition; - - public NewCaseOperatorNode(Node switchCondition, List> conditions, List> expressions, - Node elseExpr) { - this.switchCondition = switchCondition; - this.conditions = conditions; - this.expressions = expressions; - this.elseExpr = elseExpr; - if (conditions.size() != expressions.size()) { - throw new IllegalArgumentException(); - } - } - - public Node getSwitchCondition() { - return switchCondition; - } - - public List> getConditions() { - return conditions; - } - - public List> getExpressions() { - return expressions; - } - - public Node getElseExpr() { - return elseExpr; - } - -} diff --git a/src/sqlancer/ast/newast/NewFunctionNode.java b/src/sqlancer/ast/newast/NewFunctionNode.java deleted file mode 100644 index fc54941a8..000000000 --- a/src/sqlancer/ast/newast/NewFunctionNode.java +++ /dev/null @@ -1,23 +0,0 @@ -package sqlancer.ast.newast; - -import java.util.List; - -public class NewFunctionNode implements Node { - - protected List> args; - protected F func; - - public NewFunctionNode(List> args, F func) { - this.args = args; - this.func = func; - } - - public List> getArgs() { - return args; - } - - public F getFunc() { - return func; - } - -} diff --git a/src/sqlancer/ast/newast/NewInOperatorNode.java b/src/sqlancer/ast/newast/NewInOperatorNode.java deleted file mode 100644 index 0eb88c81a..000000000 --- a/src/sqlancer/ast/newast/NewInOperatorNode.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.ast.newast; - -import java.util.List; - -public class NewInOperatorNode implements Node { - - private final Node left; - private final List> right; - private final boolean isNegated; - - public NewInOperatorNode(Node left, List> right, boolean isNegated) { - this.left = left; - this.right = right; - this.isNegated = isNegated; - } - - public Node getLeft() { - return left; - } - - public List> getRight() { - return right; - } - - public boolean isNegated() { - return isNegated; - } - -} diff --git a/src/sqlancer/ast/newast/NewPostfixTextNode.java b/src/sqlancer/ast/newast/NewPostfixTextNode.java deleted file mode 100644 index 270d191ff..000000000 --- a/src/sqlancer/ast/newast/NewPostfixTextNode.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.ast.newast; - -public class NewPostfixTextNode implements Node { - - private final Node expr; - private final String text; - - public NewPostfixTextNode(Node expr, String text) { - this.expr = expr; - this.text = text; - } - - public Node getExpr() { - return expr; - } - - public String getText() { - return text; - } -} diff --git a/src/sqlancer/ast/newast/NewUnaryPostfixOperatorNode.java b/src/sqlancer/ast/newast/NewUnaryPostfixOperatorNode.java deleted file mode 100644 index d02bfbaca..000000000 --- a/src/sqlancer/ast/newast/NewUnaryPostfixOperatorNode.java +++ /dev/null @@ -1,23 +0,0 @@ -package sqlancer.ast.newast; - -import sqlancer.ast.BinaryOperatorNode.Operator; - -public class NewUnaryPostfixOperatorNode implements Node { - - protected final Operator op; - private final Node expr; - - public NewUnaryPostfixOperatorNode(Node expr, Operator op) { - this.expr = expr; - this.op = op; - } - - public String getOperatorRepresentation() { - return op.getTextRepresentation(); - } - - public Node getExpr() { - return expr; - } - -} diff --git a/src/sqlancer/ast/newast/NewUnaryPrefixOperatorNode.java b/src/sqlancer/ast/newast/NewUnaryPrefixOperatorNode.java deleted file mode 100644 index 87fa0b8f7..000000000 --- a/src/sqlancer/ast/newast/NewUnaryPrefixOperatorNode.java +++ /dev/null @@ -1,23 +0,0 @@ -package sqlancer.ast.newast; - -import sqlancer.ast.BinaryOperatorNode.Operator; - -public class NewUnaryPrefixOperatorNode implements Node { - - protected final Operator op; - private final Node expr; - - public NewUnaryPrefixOperatorNode(Node expr, Operator op) { - this.expr = expr; - this.op = op; - } - - public String getOperatorRepresentation() { - return op.getTextRepresentation(); - } - - public Node getExpr() { - return expr; - } - -} diff --git a/src/sqlancer/ast/newast/Node.java b/src/sqlancer/ast/newast/Node.java deleted file mode 100644 index ae45e1e43..000000000 --- a/src/sqlancer/ast/newast/Node.java +++ /dev/null @@ -1,5 +0,0 @@ -package sqlancer.ast.newast; - -public interface Node { - -} diff --git a/src/sqlancer/ast/newast/TableReferenceNode.java b/src/sqlancer/ast/newast/TableReferenceNode.java deleted file mode 100644 index b1b59f016..000000000 --- a/src/sqlancer/ast/newast/TableReferenceNode.java +++ /dev/null @@ -1,17 +0,0 @@ -package sqlancer.ast.newast; - -import sqlancer.schema.AbstractTable; - -public class TableReferenceNode> implements Node { - - private final T t; - - public TableReferenceNode(T table) { - this.t = table; - } - - public T getTable() { - return t; - } - -} diff --git a/src/sqlancer/citus/CitusBugs.java b/src/sqlancer/citus/CitusBugs.java new file mode 100644 index 000000000..1fda3be6a --- /dev/null +++ b/src/sqlancer/citus/CitusBugs.java @@ -0,0 +1,42 @@ +package sqlancer.citus; + +// do not make the fields final to avoid warnings +public final class CitusBugs { + + // https://github.com/citusdata/citus/issues/3987 + public static boolean bug3987; + + // https://github.com/citusdata/citus/issues/3980 + public static boolean bug3980; + + // https://github.com/citusdata/citus/issues/3957 + public static boolean bug3957; + + // https://github.com/citusdata/citus/issues/4019 + public static boolean bug4019 = true; + + // https://github.com/citusdata/citus/issues/4013 + public static boolean bug4013 = true; + + // https://github.com/citusdata/citus/issues/3982 + public static boolean bug3982 = true; + + // https://github.com/citusdata/citus/issues/3981 + public static boolean bug3981 = true; + + // https://github.com/citusdata/citus/issues/4014 + public static boolean bug4014 = true; + + // https://github.com/citusdata/citus/issues/4079 + public static boolean bug4079 = true; + + // https://github.com/citusdata/citus/issues/6298 + public static boolean bug6298 = true; + + // https://github.com/citusdata/citus/issues/8553 + public static boolean bug8553 = true; + + private CitusBugs() { + } + +} diff --git a/src/sqlancer/citus/CitusGlobalState.java b/src/sqlancer/citus/CitusGlobalState.java new file mode 100644 index 000000000..28bf193a6 --- /dev/null +++ b/src/sqlancer/citus/CitusGlobalState.java @@ -0,0 +1,24 @@ +package sqlancer.citus; + +import java.sql.SQLException; + +import sqlancer.postgres.PostgresGlobalState; + +public class CitusGlobalState extends PostgresGlobalState { + + private boolean repartition; + + public void setRepartition(boolean repartition) { + this.repartition = repartition; + } + + public boolean getRepartition() { + return repartition; + } + + @Override + public CitusSchema readSchema() throws SQLException { + return CitusSchema.fromConnection(getConnection(), getDatabaseName()); + } + +} diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java new file mode 100644 index 000000000..f2d1b0abc --- /dev/null +++ b/src/sqlancer/citus/CitusOptions.java @@ -0,0 +1,18 @@ +package sqlancer.citus; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; + +import sqlancer.postgres.PostgresOptions; + +public class CitusOptions extends PostgresOptions { + + @Parameter(names = "--repartition", description = "Specifies whether repartition joins should be allowed", arity = 1) + public boolean repartition = true; + + @Parameter(names = "--citusoracle", description = "Specifies which test oracle should be used for Citus extension to PostgreSQL") + public List citusOracle = Arrays.asList(CitusOracleFactory.QUERY_PARTITIONING); + +} diff --git a/src/sqlancer/citus/CitusOracleFactory.java b/src/sqlancer/citus/CitusOracleFactory.java new file mode 100644 index 000000000..b6e0a7f3e --- /dev/null +++ b/src/sqlancer/citus/CitusOracleFactory.java @@ -0,0 +1,70 @@ +package sqlancer.citus; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; +import sqlancer.citus.oracle.tlp.CitusTLPHavingOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; + +public enum CitusOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).with(CitusCommon.getCitusErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + HAVING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + return new CitusTLPHavingOracle(citusGlobalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(citusGlobalState)); + oracles.add(HAVING.create(citusGlobalState)); + oracles.add(new CitusTLPAggregateOracle(citusGlobalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java new file mode 100644 index 000000000..a11424f18 --- /dev/null +++ b/src/sqlancer/citus/CitusProvider.java @@ -0,0 +1,477 @@ +package sqlancer.citus; + +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.StatementExecutor; +import sqlancer.citus.gen.CitusAlterTableGenerator; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.citus.gen.CitusDeleteGenerator; +import sqlancer.citus.gen.CitusDiscardGenerator; +import sqlancer.citus.gen.CitusIndexGenerator; +import sqlancer.citus.gen.CitusInsertGenerator; +import sqlancer.citus.gen.CitusReindexGenerator; +import sqlancer.citus.gen.CitusSetGenerator; +import sqlancer.citus.gen.CitusTableGenerator; +import sqlancer.citus.gen.CitusTruncateGenerator; +import sqlancer.citus.gen.CitusUpdateGenerator; +import sqlancer.citus.gen.CitusVacuumGenerator; +import sqlancer.citus.gen.CitusViewGenerator; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.PostgresProvider; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; +import sqlancer.postgres.gen.PostgresAnalyzeGenerator; +import sqlancer.postgres.gen.PostgresClusterGenerator; +import sqlancer.postgres.gen.PostgresCommentGenerator; +import sqlancer.postgres.gen.PostgresDropIndexGenerator; +import sqlancer.postgres.gen.PostgresNotifyGenerator; +import sqlancer.postgres.gen.PostgresSequenceGenerator; +import sqlancer.postgres.gen.PostgresStatisticsGenerator; +import sqlancer.postgres.gen.PostgresTransactionGenerator; + +@AutoService(DatabaseProvider.class) +public class CitusProvider extends PostgresProvider { + + @SuppressWarnings("unchecked") + public CitusProvider() { + super((Class) (Object) CitusGlobalState.class, + (Class) (Object) CitusOptions.class); + } + + public enum Action implements AbstractAction { + ANALYZE(PostgresAnalyzeGenerator::create), // + ALTER_TABLE(g -> CitusAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g, + generateOnlyKnown)), // + CLUSTER(PostgresClusterGenerator::create), // + COMMIT(g -> { + SQLQueryAdapter query; + if (Randomly.getBoolean()) { + query = new SQLQueryAdapter("COMMIT", true); + } else if (Randomly.getBoolean()) { + query = PostgresTransactionGenerator.executeBegin(); + } else { + query = new SQLQueryAdapter("ROLLBACK", true); + } + return query; + }), // + CREATE_STATISTICS(PostgresStatisticsGenerator::insert), // + DROP_STATISTICS(PostgresStatisticsGenerator::remove), // + DELETE(CitusDeleteGenerator::create), // + DISCARD(CitusDiscardGenerator::create), // + DROP_INDEX(PostgresDropIndexGenerator::create), // + INSERT(CitusInsertGenerator::insert), // + UPDATE(CitusUpdateGenerator::create), // + TRUNCATE(CitusTruncateGenerator::create), // + VACUUM(CitusVacuumGenerator::create), // + REINDEX(CitusReindexGenerator::create), // + SET(CitusSetGenerator::create), // + CREATE_INDEX(CitusIndexGenerator::generate), // + SET_CONSTRAINTS((g) -> { + StringBuilder sb = new StringBuilder(); + sb.append("SET CONSTRAINTS ALL "); + sb.append(Randomly.fromOptions("DEFERRED", "IMMEDIATE")); + return new SQLQueryAdapter(sb.toString()); + }), // + RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // + COMMENT_ON(PostgresCommentGenerator::generate), // + RESET((g) -> new SQLQueryAdapter("RESET ALL") /* + * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also + * configuration parameter + */), // + NOTIFY(PostgresNotifyGenerator::createNotify), // + LISTEN((g) -> PostgresNotifyGenerator.createListen()), // + UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // + CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // + CREATE_VIEW(CitusViewGenerator::create); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(PostgresGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(PostgresGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + switch (a) { + case CREATE_INDEX: + case CLUSTER: + nrPerformed = r.getInteger(0, 3); + break; + case CREATE_STATISTICS: + nrPerformed = r.getInteger(0, 5); + break; + case DISCARD: + case DROP_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case COMMIT: + nrPerformed = r.getInteger(0, 0); + break; + case ALTER_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case REINDEX: + case RESET: + nrPerformed = r.getInteger(0, 3); + break; + case DELETE: + case RESET_ROLE: + case SET: + nrPerformed = r.getInteger(0, 5); + break; + case ANALYZE: + nrPerformed = r.getInteger(0, 3); + break; + case VACUUM: + case SET_CONSTRAINTS: + case COMMENT_ON: + case NOTIFY: + case LISTEN: + case UNLISTEN: + case CREATE_SEQUENCE: + case DROP_STATISTICS: + case TRUNCATE: + nrPerformed = r.getInteger(0, 2); + break; + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + + } + + private class CitusWorkerNode { + + private final String host; + private final int port; + + CitusWorkerNode(String nodeHost, int nodePort) { + this.host = nodeHost; + this.port = nodePort; + } + + public String getHost() { + return this.host; + } + + public int getPort() { + return this.port; + } + + } + + private static void distributeTable(List columns, String tableName, CitusGlobalState globalState) + throws Exception { + if (!columns.isEmpty()) { + PostgresColumn columnToDistribute = Randomly.fromList(columns); + String queryString = "SELECT create_distributed_table('" + tableName + "', '" + columnToDistribute.getName() + + "');"; + SQLQueryAdapter query = new SQLQueryAdapter(queryString, getCitusErrors()); + globalState.executeStatement(query, "SELECT create_distributed_table(?, ?);", tableName, + columnToDistribute.getName()); + } + } + + private static List getTableConstraints(String tableName, CitusGlobalState globalState) + throws SQLException { + List constraints = new ArrayList<>(); + String queryString = "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = '" + + tableName + + "' AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');"; + SQLQueryAdapter query = new SQLQueryAdapter(queryString); + SQLancerResultSet rs = query.executeAndGet(globalState, + "SELECT constraint_type FROM information_schema.table_constraints WHERE table_name = ? AND (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' or constraint_type = 'EXCLUDE');", + tableName); + while (rs.next()) { + constraints.add(rs.getString(1)); + } + return constraints; + } + + private static void createDistributedTable(String tableName, CitusGlobalState globalState) throws Exception { + List columns = new ArrayList<>(); + List tableConstraints = getTableConstraints(tableName, globalState); + if (tableConstraints.isEmpty()) { + String queryString = "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = '" + + tableName + "';"; + SQLQueryAdapter query = new SQLQueryAdapter(queryString); + SQLancerResultSet rs = query.executeAndGet(globalState, + "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ?;", tableName); + while (rs.next()) { + String columnName = rs.getString(1); + String dataType = rs.getString(2); + if (dataTypeHasDefaultOperatorForPartition(dataType)) { + PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); + columns.add(c); + } + } + } else { + HashMap> columnConstraints = new HashMap<>(); + String queryString = "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = '" + + tableName + "';"; + SQLQueryAdapter query = new SQLQueryAdapter(queryString); + SQLancerResultSet rs = query.executeAndGet(globalState, + "SELECT c.column_name, c.data_type, tc.constraint_type FROM information_schema.table_constraints tc JOIN information_schema.constraint_column_usage AS ccu USING (constraint_schema, constraint_name) JOIN information_schema.columns AS c ON c.table_schema = tc.constraint_schema AND tc.table_name = c.table_name AND ccu.column_name = c.column_name WHERE (constraint_type = 'PRIMARY KEY' OR constraint_type = 'UNIQUE' OR constraint_type = 'EXCLUDE') AND c.table_name = ?;", + tableName); + while (rs.next()) { + String columnName = rs.getString(1); + String dataType = rs.getString(2); + String constraintType = rs.getString(3); + if (dataTypeHasDefaultOperatorForPartition(dataType)) { + PostgresColumn c = new PostgresColumn(columnName, PostgresSchema.getColumnType(dataType)); + if (columnConstraints.containsKey(c)) { + columnConstraints.get(c).add(constraintType); + } else { + columnConstraints.put(c, new ArrayList<>(Arrays.asList(constraintType))); + } + } + } + for (PostgresColumn c : columnConstraints.keySet()) { + // check if all table contraints are included in column constraints, i.e. column eligible to distribute + if (tableConstraints.size() == columnConstraints.get(c).size()) { + columns.add(c); + } + } + } + distributeTable(columns, tableName, globalState); + } + + @Override + protected void createTables(PostgresGlobalState globalState, int numTables) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = CitusTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + globalState.executeStatement(createTable); + } catch (IgnoreMeException e) { + + } + } + } + + @Override + public void generateDatabase(PostgresGlobalState globalState) throws Exception { + readFunctions(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + for (PostgresTable table : globalState.getSchema().getDatabaseTables()) { + if (!(table.getTableType() == TableType.TEMPORARY || Randomly.getBooleanWithRatherLowProbability())) { + if (Randomly.getBooleanWithRatherLowProbability()) { + // create reference table + String queryString = "SELECT create_reference_table('" + table.getName() + "');"; + SQLQueryAdapter query = new SQLQueryAdapter(queryString, getCitusErrors()); + globalState.executeStatement(query, "SELECT create_reference_table(?);", table.getName()); + } else { + // create distributed table + createDistributedTable(table.getName(), (CitusGlobalState) globalState); + } + } + // else: keep local table + } + globalState.updateSchema(); + prepareTables(globalState); + if (((CitusGlobalState) globalState).getRepartition()) { + // allow repartition joins + globalState.executeStatement( + new SQLQueryAdapter("SET citus.enable_repartition_joins to ON;\n", getCitusErrors())); + } + } + + @Override + protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { + List> oracles = ((CitusOptions) globalState + .getDbmsSpecificOptions()).citusOracle.stream().map(o -> { + try { + return o.create(globalState); + } catch (Exception e1) { + throw new AssertionError(e1); + } + }).collect(Collectors.toList()); + return new CompositeTestOracle(oracles, globalState); + } + + private List readCitusWorkerNodes(PostgresGlobalState globalState, SQLConnection con) + throws SQLException { + globalState.getState().logStatement("SELECT * FROM citus_get_active_worker_nodes()"); + List citusWorkerNodes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery("SELECT * FROM citus_get_active_worker_nodes();"); + while (rs.next()) { + String nodeHost = rs.getString("node_name"); + int nodePort = rs.getInt("node_port"); + CitusWorkerNode w = new CitusWorkerNode(nodeHost, nodePort); + citusWorkerNodes.add(w); + } + } + return citusWorkerNodes; + } + + private void addCitusExtension(PostgresGlobalState globalState, SQLConnection con) throws SQLException { + globalState.getState().logStatement("CREATE EXTENSION citus;"); + try (Statement s = con.createStatement()) { + s.execute("CREATE EXTENSION citus;"); + } + } + + private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes, + int databaseIndex, String entryDatabaseName) throws SQLException { + for (CitusWorkerNode w : citusWorkerNodes) { + // connect to worker node, entry database + int hostIndex = entryURL.indexOf(host); + String preHost = entryURL.substring(0, hostIndex); + String postHost = entryURL.substring(databaseIndex - 1); + String entryWorkerURL = preHost + w.getHost() + ":" + w.getPort() + postHost; + globalState.getState().logStatement("\\q"); + globalState.getState().logStatement(entryWorkerURL); + SQLConnection con = new SQLConnection( + DriverManager.getConnection("jdbc:" + entryWorkerURL, username, password)); + + // create test database at worker node + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + // Disconnects all clients accessing `databaseName`, except the current + // https://stackoverflow.com/questions/5108876/kill-a-postgresql-session-connection + s.execute("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '" + + databaseName + "' AND pid <> pg_backend_pid()"); + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + + // connect to worker node, test database + int databaseIndexWorker = entryWorkerURL.indexOf(entryPath) + 1; + String preDatabaseNameWorker = entryWorkerURL.substring(0, databaseIndexWorker); + String postDatabaseNameWorker = entryWorkerURL.substring(databaseIndexWorker + entryDatabaseName.length()); + String testWorkerURL = preDatabaseNameWorker + databaseName + postDatabaseNameWorker; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + con = new SQLConnection(DriverManager.getConnection("jdbc:" + testWorkerURL, username, password)); + + // add citus extension to worker node, test database + addCitusExtension(globalState, con); + con.close(); + } + } + + private void addCitusWorkerNodes(PostgresGlobalState globalState, SQLConnection con, + List citusWorkerNodes) throws SQLException { + for (CitusWorkerNode w : citusWorkerNodes) { + String addWorkers = "SELECT * from citus_add_node('" + w.getHost() + "', " + w.getPort() + ");"; + globalState.getState().logStatement(addWorkers); + try (Statement s = con.createStatement()) { + s.execute(addWorkers); + } + } + } + + @SuppressWarnings("deprecation") + @Override + public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLException { + synchronized (CitusProvider.class) { + // returns connection to coordinator node, test database + SQLConnection con = super.createDatabase(globalState); + String entryDatabaseName = entryPath.substring(1); + int databaseIndex = entryURL.indexOf(entryPath) + 1; + // add citus extension to coordinator node, test database + addCitusExtension(globalState, con); + con.close(); + + // reconnect to coordinator node, entry database + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + con = new SQLConnection(DriverManager.getConnection("jdbc:" + entryURL, username, password)); + // read info about worker nodes + List citusWorkerNodes = readCitusWorkerNodes(globalState, con); + con.close(); + + // prepare worker nodes for test database + prepareCitusWorkerNodes(globalState, citusWorkerNodes, databaseIndex, entryDatabaseName); + + // reconnect to coordinator node, test database + globalState.getState().logStatement("\\q"); + globalState.getState().logStatement(testURL); + con = new SQLConnection(DriverManager.getConnection("jdbc:" + testURL, username, password)); + // add worker nodes to coordinator node for test database + addCitusWorkerNodes(globalState, con, citusWorkerNodes); + con.close(); + + // reconnect to coordinator node, test database + con = new SQLConnection(DriverManager.getConnection("jdbc:" + testURL, username, password)); + ((CitusGlobalState) globalState) + .setRepartition(((CitusOptions) globalState.getDbmsSpecificOptions()).repartition); + globalState.getState().commentStatements(); + return con; + } + } + + @Override + protected void prepareTables(PostgresGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + CitusProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 5000;\n")); + } + + @Override + public String getDBMSName() { + return "citus"; + } + + private static ExpectedErrors getCitusErrors() { + ExpectedErrors errors = new ExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return errors; + } + + private static boolean dataTypeHasDefaultOperatorForPartition(String dataType) { + return !(dataType.equals("money") || dataType.equals("bit varying")); + } + +} diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java new file mode 100644 index 000000000..b2550bdce --- /dev/null +++ b/src/sqlancer/citus/CitusSchema.java @@ -0,0 +1,92 @@ +package sqlancer.citus; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.SQLConnection; +import sqlancer.postgres.PostgresSchema; + +public class CitusSchema extends PostgresSchema { + + public CitusSchema(List databaseTables, String databaseName) { + super(new ArrayList<>(databaseTables), databaseName); + } + + public static class CitusTable extends PostgresTable { + + private PostgresColumn distributionColumn; + // colocationId is null for local tables + private Integer colocationId; + + public CitusTable(String tableName, List columns, List indexes, + TableType tableType, List statistics, boolean isView, boolean isInsertable, + PostgresColumn distributionColumn, Integer colocationId) { + super(tableName, columns, indexes, tableType, statistics, isView, isInsertable); + this.distributionColumn = distributionColumn; + this.colocationId = colocationId; + } + + public CitusTable(PostgresTable table, PostgresColumn distributionColumn, Integer colocationId) { + super(table.getName(), table.getColumns(), table.getIndexes(), table.getTableType(), table.getStatistics(), + table.isView(), table.isInsertable()); + this.distributionColumn = distributionColumn; + this.colocationId = colocationId; + } + + public void setDistributionColumn(PostgresColumn distributionColumn) { + this.distributionColumn = distributionColumn; + } + + public void setColocationId(Integer colocationId) { + this.colocationId = colocationId; + } + + public PostgresColumn getDistributionColumn() { + return this.distributionColumn; + } + + public Integer getColocationId() { + return this.colocationId; + } + + } + + public static CitusSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement(); ResultSet rs = s.executeQuery( + "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + /* skip Citus-managed views in the public schema (citus_tables, citus_schemas, etc.) */ + if (tableName.startsWith("citus_")) { + continue; + } + String distributionColumnName = rs.getString("dist_col_name"); + Integer colocationId = rs.getInt("colocationid"); + if (rs.wasNull()) { + colocationId = null; + } + PostgresTable t = schema.getDatabaseTable(tableName); + PostgresColumn distributionColumn = null; + if (t == null) { + continue; + } + if (distributionColumnName != null && !distributionColumnName.equals("")) { + distributionColumn = t.getColumns().stream().filter(c -> c.getName().equals(distributionColumnName)) + .collect(Collectors.toList()).get(0); + } + CitusTable tCitus = new CitusTable(t, distributionColumn, colocationId); + databaseTables.add(tCitus); + } + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + return new CitusSchema(databaseTables, databaseName); + } +} diff --git a/src/sqlancer/citus/gen/CitusAlterTableGenerator.java b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java new file mode 100644 index 000000000..f8238194d --- /dev/null +++ b/src/sqlancer/citus/gen/CitusAlterTableGenerator.java @@ -0,0 +1,50 @@ +package sqlancer.citus.gen; + +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.gen.PostgresAlterTableGenerator; + +public class CitusAlterTableGenerator extends PostgresAlterTableGenerator { + + public CitusAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, + boolean generateOnlyKnown) { + super(randomTable, globalState, generateOnlyKnown); + } + + public static SQLQueryAdapter create(PostgresTable randomTable, PostgresGlobalState globalState, + boolean generateOnlyKnown) { + return new CitusAlterTableGenerator(randomTable, globalState, generateOnlyKnown).generate(); + } + + @Override + public List getActions(ExpectedErrors errors) { + List action = super.getActions(errors); + CitusCommon.addCitusErrors(errors); + action.remove(Action.ALTER_COLUMN_SET_STATISTICS); + action.remove(Action.ALTER_COLUMN_SET_ATTRIBUTE_OPTION); + action.remove(Action.ALTER_COLUMN_RESET_ATTRIBUTE_OPTION); + action.remove(Action.ALTER_COLUMN_SET_STORAGE); + action.remove(Action.DISABLE_ROW_LEVEL_SECURITY); + action.remove(Action.ENABLE_ROW_LEVEL_SECURITY); + action.remove(Action.FORCE_ROW_LEVEL_SECURITY); + action.remove(Action.NO_FORCE_ROW_LEVEL_SECURITY); + action.remove(Action.CLUSTER_ON); + action.remove(Action.SET_WITHOUT_CLUSTER); + action.remove(Action.SET_WITH_OIDS); + action.remove(Action.SET_WITHOUT_OIDS); + action.remove(Action.SET_LOGGED_UNLOGGED); + action.remove(Action.NOT_OF); + action.remove(Action.OWNER_TO); + action.remove(Action.REPLICA_IDENTITY); + if (action.isEmpty()) { + throw new IgnoreMeException(); + } + return action; + } + +} diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java new file mode 100644 index 000000000..58b1b7c16 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -0,0 +1,110 @@ +package sqlancer.citus.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.citus.CitusBugs; +import sqlancer.common.query.ExpectedErrors; + +public final class CitusCommon { + + private CitusCommon() { + } + + public static List getCitusErrors() { + // not supported by Citus + ArrayList errors = new ArrayList<>(); + errors.add("failed to evaluate partition key in insert"); + errors.add("cannot perform an INSERT without a partition column value"); + errors.add("cannot perform an INSERT with NULL in the partition column"); + errors.add("recursive CTEs are not supported in distributed queries"); + errors.add("recursive CTEs are only supported when they contain a filter on the distribution column"); + errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); + errors.add("Subqueries in HAVING cannot refer to outer query"); + errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); + errors.add("functions used in UPDATE queries on distributed tables must not be VOLATILE"); + errors.add("STABLE functions used in UPDATE queries cannot be called with column references"); + errors.add("of modification queries on distributed tables must not be VOLATILE"); + errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); + errors.add("cannot execute ALTER TABLE command involving partition column"); + errors.add("alter table command is currently unsupported"); + errors.add("on distributed partitioned tables are not supported"); + errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); + errors.add("is not a regular, foreign or partitioned table"); + errors.add("must be a distributed table or a reference table"); + errors.add("creating unique indexes on non-partition columns is currently unsupported"); + errors.add("modifying the partition value of rows is not allowed"); + errors.add("creating unique indexes on non-partition columns is currently unsupported"); + errors.add("Distributed relations must not use GENERATED ... AS IDENTITY"); + errors.add("cannot drop multiple distributed objects in a single command"); + errors.add("is not distributed"); + errors.add("cannot create constraint on"); + errors.add("cannot create foreign key constraint"); // SET NULL or SET DEFAULT is not supported in ON DELETE + // operation when distribution key is included in the + // foreign key constraint + errors.add("cannot modify views over distributed tables"); + + // not supported by Citus (restrictions on SELECT queries) + errors.add( + "complex joins are only supported when all distributed tables are co-located and joined on their distribution columns"); + errors.add( + "complex joins are only supported when all distributed tables are joined on their distribution columns with equal operator"); + errors.add("cannot perform distributed planning on this query"); + errors.add("cannot pushdown the subquery"); + // see https://github.com/sqlancer/sqlancer/issues/215 + errors.add("direct joins between distributed and local tables are not supported"); + errors.add("unlogged columnar tables are not supported"); + errors.add("UPDATE and CTID scans not supported for ColumnarScan"); + errors.add("unsupported access method for the index on columnar table"); + errors.add("BRIN indexes on columnar tables are not supported"); + errors.add("invalid byte sequence for encoding \"UTF8\": 0x00"); + errors.add("columnar_tuple_insert_speculative not implemented"); + errors.add("row field count is 1, expected 2"); + errors.add("incorrect binary data format"); + errors.add("invalid sign in external \"numeric\" value"); + errors.add("Foreign keys and AFTER ROW triggers are not supported for columnar tables"); + errors.addAll(getColumnarOidErrors()); + + // current errors in Citus (to be removed once fixed) + if (CitusBugs.bug3957) { + errors.add("unrecognized node type: 127"); + } + if (CitusBugs.bug3980 || CitusBugs.bug3987 || CitusBugs.bug4019) { + errors.add("syntax error at or near"); + } + if (CitusBugs.bug3982) { + errors.add("failed to find conversion function from unknown to text"); + errors.add("invalid input syntax for"); + } + if (CitusBugs.bug4013) { + errors.add("ERROR: LIMIT must not be negative"); + } + if (CitusBugs.bug3981) { + errors.add("value too long for type"); + } + if (CitusBugs.bug4014) { + errors.add("is ambiguous"); + } + if (CitusBugs.bug4079) { + errors.add("aggregate function calls cannot be nested"); + } + return errors; + } + + /** + * Citus can fail with "could not open relation with OID 0" when operating on columnar temporary tables (e.g., USING + * columnar ON COMMIT DROP), during VACUUM, DISCARD TEMPORARY, or INSERT operations where Citus cannot resolve the + * relation OID. + * + * @return the list of expected error substrings for columnar OID resolution failures. + */ + public static List getColumnarOidErrors() { + List errors = new ArrayList<>(); + errors.add("could not open relation with OID 0"); + return errors; + } + + public static void addCitusErrors(ExpectedErrors errors) { + errors.addAll(getCitusErrors()); + } +} diff --git a/src/sqlancer/citus/gen/CitusDeleteGenerator.java b/src/sqlancer/citus/gen/CitusDeleteGenerator.java new file mode 100644 index 000000000..925a070bb --- /dev/null +++ b/src/sqlancer/citus/gen/CitusDeleteGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresDeleteGenerator; + +public final class CitusDeleteGenerator { + + private CitusDeleteGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter deleteQuery = PostgresDeleteGenerator.create(globalState); + ExpectedErrors errors = deleteQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return deleteQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusDiscardGenerator.java b/src/sqlancer/citus/gen/CitusDiscardGenerator.java new file mode 100644 index 000000000..f4a1b3240 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusDiscardGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresDiscardGenerator; + +public final class CitusDiscardGenerator { + + private CitusDiscardGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter discardQuery = PostgresDiscardGenerator.create(globalState); + ExpectedErrors errors = discardQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return discardQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusIndexGenerator.java b/src/sqlancer/citus/gen/CitusIndexGenerator.java new file mode 100644 index 000000000..84b96e2cd --- /dev/null +++ b/src/sqlancer/citus/gen/CitusIndexGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresInsertGenerator; + +public final class CitusIndexGenerator { + + private CitusIndexGenerator() { + } + + public static SQLQueryAdapter generate(PostgresGlobalState globalState) { + SQLQueryAdapter createIndexQuery = PostgresInsertGenerator.insert(globalState); + ExpectedErrors errors = createIndexQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return createIndexQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusInsertGenerator.java b/src/sqlancer/citus/gen/CitusInsertGenerator.java new file mode 100644 index 000000000..78b5794e7 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusInsertGenerator.java @@ -0,0 +1,24 @@ +package sqlancer.citus.gen; + +import sqlancer.citus.CitusBugs; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresInsertGenerator; + +public final class CitusInsertGenerator { + + private CitusInsertGenerator() { + } + + public static SQLQueryAdapter insert(PostgresGlobalState globalState) { + SQLQueryAdapter insertQuery = PostgresInsertGenerator.insert(globalState); + ExpectedErrors errors = insertQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + if (CitusBugs.bug6298) { + errors.add("columnar_tuple_insert_speculative not implemented"); + } + return insertQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusReindexGenerator.java b/src/sqlancer/citus/gen/CitusReindexGenerator.java new file mode 100644 index 000000000..6f37cbe06 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusReindexGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresReindexGenerator; + +public final class CitusReindexGenerator { + + private CitusReindexGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter reindexQuery = PostgresReindexGenerator.create(globalState); + ExpectedErrors errors = reindexQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return reindexQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusSetGenerator.java b/src/sqlancer/citus/gen/CitusSetGenerator.java new file mode 100644 index 000000000..03304fece --- /dev/null +++ b/src/sqlancer/citus/gen/CitusSetGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresSetGenerator; + +public final class CitusSetGenerator { + + private CitusSetGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter setQuery = PostgresSetGenerator.create(globalState); + ExpectedErrors errors = setQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return setQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java new file mode 100644 index 000000000..5d6b8b249 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -0,0 +1,32 @@ +package sqlancer.citus.gen; + +import sqlancer.citus.CitusBugs; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.gen.PostgresTableGenerator; + +public class CitusTableGenerator extends PostgresTableGenerator { + + public CitusTableGenerator(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, + PostgresGlobalState globalState) { + super(tableName, newSchema, generateOnlyKnown, globalState); + CitusCommon.addCitusErrors(errors); + errors.add("columnar_parallelscan_estimate not implemented"); // see + // https://github.com/sqlancer/sqlancer/issues/402 + } + + public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, + PostgresGlobalState globalState) { + return new CitusTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); + } + + @Override + protected void generateInherits() { + if (CitusBugs.bug8553) { + return; + } + super.generateInherits(); + } + +} diff --git a/src/sqlancer/citus/gen/CitusTruncateGenerator.java b/src/sqlancer/citus/gen/CitusTruncateGenerator.java new file mode 100644 index 000000000..cf36ce9c2 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusTruncateGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresTruncateGenerator; + +public final class CitusTruncateGenerator { + + private CitusTruncateGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter truncateQuery = PostgresTruncateGenerator.create(globalState); + ExpectedErrors errors = truncateQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return truncateQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusUpdateGenerator.java b/src/sqlancer/citus/gen/CitusUpdateGenerator.java new file mode 100644 index 000000000..1429c1484 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusUpdateGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresUpdateGenerator; + +public final class CitusUpdateGenerator { + + private CitusUpdateGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter updateQuery = PostgresUpdateGenerator.create(globalState); + ExpectedErrors errors = updateQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return updateQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusVacuumGenerator.java b/src/sqlancer/citus/gen/CitusVacuumGenerator.java new file mode 100644 index 000000000..ae73dbf82 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusVacuumGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresVacuumGenerator; + +public final class CitusVacuumGenerator { + + private CitusVacuumGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter vacuumQuery = PostgresVacuumGenerator.create(globalState); + ExpectedErrors errors = vacuumQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return vacuumQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusViewGenerator.java b/src/sqlancer/citus/gen/CitusViewGenerator.java new file mode 100644 index 000000000..a2db0cf67 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusViewGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresViewGenerator; + +public final class CitusViewGenerator { + + private CitusViewGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter viewQuery = PostgresViewGenerator.create(globalState); + ExpectedErrors errors = viewQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return viewQuery; + } + +} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java new file mode 100644 index 000000000..1ab27ab85 --- /dev/null +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPAggregateOracle.java @@ -0,0 +1,36 @@ +package sqlancer.citus.oracle.tlp; + +import java.sql.SQLException; +import java.util.Arrays; + +import sqlancer.citus.CitusGlobalState; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; + +public class CitusTLPAggregateOracle extends PostgresTLPAggregateOracle { + + private final CitusTLPBase citusTLPBase; + + public CitusTLPAggregateOracle(CitusGlobalState state) { + super(state); + CitusCommon.addCitusErrors(errors); + citusTLPBase = new CitusTLPBase(state); + } + + @Override + public void check() throws SQLException { + state.setAllowedFunctionTypes(Arrays.asList(PostgresGlobalState.IMMUTABLE)); + citusTLPBase.check(); + s = citusTLPBase.getSchema(); + targetTables = citusTLPBase.getTargetTables(); + gen = citusTLPBase.getGenerator(); + select = citusTLPBase.getSelect(); + predicate = citusTLPBase.getPredicate(); + negatedPredicate = citusTLPBase.getNegatedPredicate(); + isNullPredicate = citusTLPBase.getIsNullPredicate(); + aggregateCheck(); + state.setDefaultAllowedFunctionTypes(); + } + +} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java new file mode 100644 index 000000000..8512ed184 --- /dev/null +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPBase.java @@ -0,0 +1,225 @@ +package sqlancer.citus.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.citus.CitusGlobalState; +import sqlancer.citus.CitusSchema.CitusTable; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.ast.PostgresBinaryComparisonOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; +import sqlancer.postgres.ast.PostgresColumnValue; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.tlp.PostgresTLPBase; + +public class CitusTLPBase extends PostgresTLPBase { + + Map distributedTables; + List referenceTables; + List localTables; + + public CitusTLPBase(CitusGlobalState state) { + super(state); + CitusCommon.addCitusErrors(errors); + } + + public PostgresSchema getSchema() { + return s; + } + + public PostgresTables getTargetTables() { + return targetTables; + } + + public PostgresExpressionGenerator getGenerator() { + return gen; + } + + public PostgresSelect getSelect() { + return select; + } + + public PostgresExpression getPredicate() { + return predicate; + } + + public PostgresExpression getNegatedPredicate() { + return negatedPredicate; + } + + public PostgresExpression getIsNullPredicate() { + return isNullPredicate; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + storeCitusTableTypes(); + List tables = new ArrayList<>(); + List joins = generateJoins(tables); + generateSelectBase(tables, joins); + } + + private List generateJoins(List tables) { + List joins = null; + tables.clear(); + if (distributedTables.isEmpty() + || !referenceTables.isEmpty() && !Randomly.getBooleanWithRatherLowProbability()) { + if (!localTables.isEmpty()) { + // joins including only local tables + // supports complex joins + targetTables = new PostgresTables(Randomly.nonEmptySubset(localTables)); + } + if (!referenceTables.isEmpty()) { + // joins including reference tables + // supports complex joins + List targetTableList = new ArrayList<>(referenceTables); + if (!distributedTables.isEmpty()) { + // joins including distributed and reference tables + // supports complex joins + targetTableList.add(Randomly.fromList(new ArrayList<>(distributedTables.keySet()))); + } + targetTables = new PostgresTables(Randomly.nonEmptySubset(targetTableList)); + } + tables.addAll(targetTables.getTables()); + joins = getJoinStatements(state, targetTables.getColumns(), tables); + } else { + // joins between distributed tables + // join including distribution columns + // supports complex joins if colocated + tables.addAll(Randomly.nonEmptySubset(new ArrayList<>(distributedTables.keySet()))); + targetTables = new PostgresTables(tables); + CitusTable fromTable = (CitusTable) Randomly.fromList(tables); + joins = getCitusJoinStatements((CitusGlobalState) state, tables, fromTable); + if (Randomly.getBooleanWithRatherLowProbability() && !localTables.isEmpty()) { + addSubqueryJoinStatements(state, joins, fromTable); + } + } + return joins; + } + + private void storeCitusTableTypes() { + distributedTables = new HashMap<>(); + referenceTables = new ArrayList<>(); + localTables = new ArrayList<>(); + for (PostgresTable table : s.getDatabaseTables()) { + CitusTable citusTable = (CitusTable) table; + Integer colocationId = citusTable.getColocationId(); + PostgresColumn distributionColumn = citusTable.getDistributionColumn(); + if (colocationId != null && distributionColumn != null) { + distributedTables.put(citusTable, colocationId); + } else if (colocationId != null) { + referenceTables.add(citusTable); + } else { + localTables.add(citusTable); + } + } + } + + private PostgresJoin allColocatedJoins(CitusTable joinTable, CitusTable fromTable, + PostgresExpressionGenerator citusJoinGen) { + PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); + PostgresExpression rightExpr = new PostgresColumnValue(joinTable.getDistributionColumn(), null); + // JOIN over equality between the distribution columns of the tables being joined + PostgresExpression equiJoinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, + PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); + PostgresExpression joinClause = null; + if (Randomly.getBooleanWithSmallProbability()) { + // add randomly generated boolean statement to JOIN clause + joinClause = new PostgresBinaryLogicalOperation(equiJoinClause, + citusJoinGen.generateExpression(PostgresDataType.BOOLEAN), + PostgresBinaryLogicalOperation.BinaryLogicalOperator.AND); + } else { + joinClause = equiJoinClause; + } + PostgresJoinType options = Randomly.fromOptions(PostgresJoinType.INNER, PostgresJoinType.LEFT, + PostgresJoinType.RIGHT, PostgresJoinType.FULL); + return new PostgresJoin(new PostgresFromTable(joinTable, Randomly.getBoolean()), joinClause, options); + } + + private PostgresJoin repartitionJoins(CitusTable joinTable, CitusTable fromTable) { + PostgresExpression leftExpr = new PostgresColumnValue(fromTable.getDistributionColumn(), null); + List candidateRightColumns = joinTable.getColumns().stream() + .filter(c -> c.getType().equals(fromTable.getDistributionColumn().getType())) + .collect(Collectors.toList()); + if (candidateRightColumns.isEmpty()) { + return null; + } + PostgresExpression rightExpr = new PostgresColumnValue(Randomly.fromList(candidateRightColumns), null); + // JOIN over equality between the distribution column of one table and a column that matches the data type from + // the other table being joined + PostgresExpression joinClause = new PostgresBinaryComparisonOperation(leftExpr, rightExpr, + PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator.EQUALS); + PostgresJoinType options = PostgresJoinType.INNER; + return new PostgresJoin(new PostgresFromTable(joinTable, Randomly.getBoolean()), joinClause, options); + } + + List getCitusJoinStatements(CitusGlobalState globalState, List joinTables, + CitusTable fromTable) { + List columns = new ArrayList<>(); + for (PostgresTable t : joinTables) { + columns.add(((CitusTable) t).getDistributionColumn()); + } + List joinStatements = new ArrayList<>(); + PostgresExpressionGenerator citusJoinGen = new PostgresExpressionGenerator(globalState).setColumns(columns); + joinTables.remove(fromTable); + // check if all tables being joined are colocated + boolean allColocated = true; + for (PostgresTable t : joinTables) { + boolean colocated = distributedTables.get(fromTable).equals(distributedTables.get(t)); + allColocated = allColocated && colocated; + } + while (!joinTables.isEmpty()) { + CitusTable table = (CitusTable) Randomly.fromList(joinTables); + joinTables.remove(table); + PostgresJoin j = null; + if (allColocated) { + j = allColocatedJoins(table, fromTable, citusJoinGen); + // check if repartition joins are allowed if all tables are not colocated + } else if (globalState.getRepartition()) { + j = repartitionJoins(table, fromTable); + } + if (j != null) { + joinStatements.add(j); + } + } + joinTables.add(fromTable); + return joinStatements; + } + + private void addSubqueryJoinStatements(PostgresGlobalState globalState, List joinStatements, + PostgresTable fromTable) { + // JOIN with subquery + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresTables subqueryTables = new PostgresTables(Randomly.nonEmptySubset(localTables)); + List columns = new ArrayList<>(); + columns.addAll(subqueryTables.getColumns()); + columns.addAll(fromTable.getColumns()); + PostgresExpression subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); + PostgresExpressionGenerator subqueryJoinGen = new PostgresExpressionGenerator(globalState) + .setColumns(columns); + PostgresExpression joinClause = subqueryJoinGen.generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); + joinStatements.add(j); + } + + } + +} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java new file mode 100644 index 000000000..4162cb9a9 --- /dev/null +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPHavingOracle.java @@ -0,0 +1,36 @@ +package sqlancer.citus.oracle.tlp; + +import java.sql.SQLException; +import java.util.Arrays; + +import sqlancer.citus.CitusGlobalState; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; + +public class CitusTLPHavingOracle extends PostgresTLPHavingOracle { + + private final CitusTLPBase citusTLPBase; + + public CitusTLPHavingOracle(CitusGlobalState state) { + super(state); + CitusCommon.addCitusErrors(errors); + citusTLPBase = new CitusTLPBase(state); + } + + @Override + public void check() throws SQLException { + state.setAllowedFunctionTypes(Arrays.asList(PostgresGlobalState.IMMUTABLE)); + citusTLPBase.check(); + s = citusTLPBase.getSchema(); + targetTables = citusTLPBase.getTargetTables(); + gen = citusTLPBase.getGenerator(); + select = citusTLPBase.getSelect(); + predicate = citusTLPBase.getPredicate(); + negatedPredicate = citusTLPBase.getNegatedPredicate(); + isNullPredicate = citusTLPBase.getIsNullPredicate(); + havingCheck(); + state.setDefaultAllowedFunctionTypes(); + } + +} diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index db704f315..09fbe5ea8 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -1,49 +1,66 @@ package sqlancer.clickhouse; -import java.util.Set; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; public final class ClickHouseErrors { private ClickHouseErrors() { } - public static void addExpectedExpressionErrors(Set errors) { - // errors.add("Illegal type (String) of argument of function not"); - // errors.add("Illegal type String of column for constant filter. Must be UInt8 or Nullable(UInt8)"); - // errors.add("Illegal type Int32 of column for constant filter. Must be UInt8 or Nullable(UInt8)"); - // errors.add("Illegal type UInt32 of column for constant filter. Must be UInt8 or Nullable(UInt8)"); - // errors.add("Illegal type Int32 of column for filter. Must be UInt8 or Nullable(UInt8) or Const variants of - // them."); - // errors.add("Illegal type String of column for filter. Must be UInt8 or Nullable(UInt8) or Const variants of - // them."); - // errors.add("Illegal type Int64 of column for constant filter. Must be UInt8 or Nullable(UInt8)"); - errors.add("Illegal type"); - errors.add("Argument at index 1 for function like must be constant"); - errors.add("Argument at index 1 for function notLike must be constant"); - - // regex - errors.add("invalid escape sequence"); - errors.add("invalid character class range"); - errors.add("Memory limit"); - errors.add("There is no supertype for types"); - errors.add("Bad get: has Int64, requested UInt64"); - errors.add("Cannot convert string"); - errors.add("Cannot read floating point value"); - errors.add("Cannot parse infinity."); - errors.add("Attempt to read after eof: while converting"); - errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database - errors.add("is not under aggregate function"); - } + public static List getExpectedExpressionErrors() { + return List.of("Argument at index 1 for function like must be constant", + "Argument at index 1 for function notLike must be constant", + "Attempt to read after eof: while converting", "Bad get: has Int64, requested UInt64", + "Cannot convert string", "Cannot insert NULL value into a column of type", + "Cannot parse Int32 from String, because value is too short", "Cannot parse NaN.: while converting", // https://github.com/ClickHouse/ClickHouse/issues/22710 + "Cannot parse infinity.", "Cannot parse number with a sign character but without any numeric character", + "Cannot parse number with multiple sign (+/-) characters or intermediate sign character", + "Cannot parse string", "Cannot read floating point value", + "Cyclic aliases: default expression and column type are incompatible", "Directory for table data", + "Directory not empty", "Expected one of: compound identifier, identifier, list of elements (version", // VALUES + // () + "Function 'like' doesn't support search with non-constant needles in constant haystack", "Illegal type", + "Illegal value (aggregate function) for positional argument in GROUP BY", + "Invalid escape sequence at the end of LIKE pattern", "Invalid type for filter in", "Memory limit", + "OptimizedRegularExpression: cannot compile re2", "Partition key cannot contain constants", + "Positional argument out of bounds", "Sampling expression must be present in the primary key", + "Sorting key cannot contain constants", "There is no supertype for types", "argument of function", + "but its arguments considered equal according to constraints", "does not return a value of type UInt8", + "doesn't exist", // TODO: consecutive test runs can lead to dropped database + "in block. There are only columns:", // https://github.com/ClickHouse/ClickHouse/issues/42399 + "invalid character class range", "invalid escape sequence", + "is not under aggregate function and not in GROUP BY", "is not under aggregate function", + "is violated at row 1. Expression:", // TODO: check constraint on table creation + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition", + "there are only columns", "there are columns", "(NOT_FOUND_COLUMN_IN_BLOCK)", "Missing columns", + "Ambiguous column", "Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)", + "Floating point partition key is not supported", "Cannot get JOIN keys from JOIN ON section", + "ILLEGAL_DIVISION", "DECIMAL_OVERFLOW", + "Cannot convert out of range floating point value to integer type", + "Unexpected inf or nan to integer conversion", "No such name in Block::erase", // https://github.com/ClickHouse/ClickHouse/issues/42769 + "EMPTY_LIST_OF_COLUMNS_QUERIED", // https://github.com/ClickHouse/ClickHouse/issues/43003 + "EMPTY_LIST_OF_COLUMNS_PASSED", // https://github.com/ClickHouse/ClickHouse/pull/81835 + "cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)", "AMBIGUOUS_IDENTIFIER", "CYCLIC_ALIASES", + "Positional argument numeric constant expression is not representable as", + "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", + "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", + "Cannot get JOIN keys from JOIN ON section", "Unexpected inf or nan to integer conversion", + "Cannot determine join keys in", "Unsigned type must not contain", + "Unexpected inf or nan to integer conversion", - public static void addExpressionHavingErrors(Set errors) { - errors.add("Memory limit"); + // The way we generate JOINs we can have ambiguous left table column without + // alias + // We may not count it as an issue, but it makes no sense to add more complex + // AST generation logic + "MULTIPLE_EXPRESSIONS_FOR_ALIAS", "AMBIGUOUS_IDENTIFIER", // https://github.com/ClickHouse/ClickHouse/issues/45389 + "AMBIGUOUS_COLUMN_NAME", // same https://github.com/ClickHouse/ClickHouse/issues/45389 + "No equality condition found in JOIN ON expression", "Cannot parse number with multiple sign"); } - public static void addQueryErrors(Set errors) { - errors.add("Memory limit"); + public static void addExpectedExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExpressionErrors()); } - public static void addGroupingErrors(Set errors) { - errors.add("Memory limit"); - } } diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 51554710c..fec8b62a7 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -1,46 +1,30 @@ package sqlancer.clickhouse; +import java.util.Arrays; +import java.util.List; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.MainOptions; -import sqlancer.TestOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; +import sqlancer.DBMSSpecificOptions; -@Parameters(separators = "=", commandDescription = "ClickHouse") -public class ClickHouseOptions extends MainOptions { +@Parameters(separators = "=", commandDescription = "ClickHouse (default port: " + ClickHouseOptions.DEFAULT_PORT + + ", default host: " + ClickHouseOptions.DEFAULT_HOST + ")") +public class ClickHouseOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 8123; - @Parameter(names = "--oracle", converter = MainOptions.DBMSConverter.class) - public List oracle = Arrays.asList(ClickHouseOracle.TLPWhere); + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(ClickHouseOracleFactory.TLPWhere); @Parameter(names = { "--test-joins" }, description = "Allow the generation of JOIN clauses", arity = 1) public boolean testJoins = true; - public enum ClickHouseOracle { - TLPWhere { - @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPWhereOracle(globalState); - } - }, - TLPGroupBy { - @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPGroupByOracle(globalState); - } - }, - TLPHaving { - @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPHavingOracle(globalState); - } - }; - - public abstract TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException; + @Parameter(names = { "--analyzer" }, description = "Enable analyzer in ClickHouse", arity = 1) + public boolean enableAnalyzer = true; + + @Override + public List getTestOracleFactory() { + return oracle; } } diff --git a/src/sqlancer/clickhouse/ClickHouseOracleFactory.java b/src/sqlancer/clickhouse/ClickHouseOracleFactory.java new file mode 100644 index 000000000..53b38a6f1 --- /dev/null +++ b/src/sqlancer/clickhouse/ClickHouseOracleFactory.java @@ -0,0 +1,62 @@ +package sqlancer.clickhouse; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; + +public enum ClickHouseOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(ClickHouseErrors.getExpectedExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + TLPDistinct { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPDistinctOracle(globalState); + } + }, + TLPGroupBy { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPGroupByOracle(globalState); + } + }, + TLPAggregate { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPAggregateOracle(globalState); + } + }, + TLPHaving { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPHavingOracle(globalState); + } + }, + NoREC { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + + return new NoRECOracle<>(globalState, gen, errors); + } + } +} diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 7e519dca3..c611d77d7 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -1,31 +1,31 @@ package sqlancer.clickhouse; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.List; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; -import sqlancer.GlobalState; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; -import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; +import sqlancer.MainOptions; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseInsertGenerator; import sqlancer.clickhouse.gen.ClickHouseTableGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; -public class ClickHouseProvider extends ProviderAdapter { +@AutoService(DatabaseProvider.class) +public class ClickHouseProvider extends SQLProviderAdapter { public ClickHouseProvider() { super(ClickHouseGlobalState.class, ClickHouseOptions.class); @@ -35,15 +35,15 @@ public enum Action implements AbstractAction { INSERT(ClickHouseInsertGenerator::getQuery); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(ClickHouseGlobalState state) throws SQLException { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(ClickHouseGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -57,19 +57,10 @@ private static int mapActions(ClickHouseGlobalState globalState, Action a) { } } - public static class ClickHouseGlobalState extends GlobalState { + public static class ClickHouseGlobalState extends SQLGlobalState { - private ClickHouseSchema schema; private ClickHouseOptions clickHouseOptions; - public void setSchema(ClickHouseSchema schema) { - this.schema = schema; - } - - public ClickHouseSchema getSchema() { - return schema; - } - public void setClickHouseOptions(ClickHouseOptions clickHouseOptions) { this.clickHouseOptions = clickHouseOptions; } @@ -87,86 +78,57 @@ public String getOracleName() { public String getDatabaseName() { return super.getDatabaseName() + this.getOracleName(); } + + @Override + protected ClickHouseSchema readSchema() throws SQLException { + return ClickHouseSchema.fromConnection(getConnection(), getDatabaseName()); + } } @Override - public void generateAndTestDatabase(ClickHouseGlobalState globalState) throws SQLException { - StateLogger logger = globalState.getLogger(); - QueryManager manager = globalState.getManager(); - globalState - .setSchema(ClickHouseSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName())); - for (int i = 0; i < Randomly.fromOptions(1); i++) { - boolean success = false; + public void generateDatabase(ClickHouseGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2, 3, 4, 5); i++) { + boolean success; do { - Query qt = new ClickHouseTableGenerator().getQuery(globalState); - success = manager.execute(qt); - logger.writeCurrent(globalState.getState()); - globalState.setSchema( - ClickHouseSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName())); - try { - logger.getCurrentFileWriter().close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - logger.currentFileWriter = null; + String tableName = ClickHouseCommon.createTableName(i); + SQLQueryAdapter qt = ClickHouseTableGenerator.createTableStatement(tableName, globalState); + success = globalState.executeStatement(qt); } while (!success); } + // TODO: add more Actions to populate table StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), ClickHouseProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - globalState.setSchema(ClickHouseSchema.fromConnection(globalState.getConnection(), - globalState.getDatabaseName())); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } }); se.executeStatements(); - manager.incrementCreateDatabase(); - - List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (SQLException e1) { - throw new AssertionError(e1); - } - }).collect(Collectors.toList()); - CompositeTestOracle oracle = new CompositeTestOracle(oracles); - - for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { - try { - oracle.check(); - } catch (IgnoreMeException e) { - continue; - } - manager.incrementSelectQueryCount(); - } - - try { - if (globalState.getOptions().logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } } @Override - public Connection createDatabase(ClickHouseGlobalState globalState) throws SQLException { - ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); + public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQLException { + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = ClickHouseOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = ClickHouseOptions.DEFAULT_PORT; + } + + ClickHouseOptions clickHouseOptions = globalState.getDbmsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); - String url = "jdbc:clickhouse://localhost:8123/test"; + String url = String.format("jdbc:clickhouse://%s:%d/%s", host, port, "default"); String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); String dropDatabaseCommand = "DROP DATABASE IF EXISTS " + databaseName; - globalState.getState().statements.add(new QueryAdapter(dropDatabaseCommand)); + globalState.getState().logStatement(dropDatabaseCommand); String createDatabaseCommand = "CREATE DATABASE IF NOT EXISTS " + databaseName; - globalState.getState().statements.add(new QueryAdapter(createDatabaseCommand)); + globalState.getState().logStatement(createDatabaseCommand); + String useDatabaseCommand = "USE " + databaseName; // Noop. To reproduce easier. + globalState.getState().logStatement(useDatabaseCommand); try (Statement s = con.createStatement()) { s.execute(dropDatabaseCommand); Thread.sleep(1000); @@ -180,9 +142,11 @@ public Connection createDatabase(ClickHouseGlobalState globalState) throws SQLEx e.printStackTrace(); } con.close(); - con = DriverManager.getConnection("jdbc:clickhouse://localhost:18123/" + databaseName, + con = DriverManager.getConnection( + String.format("jdbc:clickhouse://%s:%d/%s?socket_timeout=300000%s", host, port, databaseName, + clickHouseOptions.enableAnalyzer ? "&allow_experimental_analyzer=1" : ""), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - return con; + return new SQLConnection(con); } @Override diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index f0c7523f5..8f8f906ec 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -1,30 +1,30 @@ package sqlancer.clickhouse; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.StateToReproduce; +import sqlancer.SQLConnection; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseConstant; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.TableIndex; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; -public class ClickHouseSchema extends AbstractSchema { +public class ClickHouseSchema extends AbstractSchema { public static class ClickHouseLancerDataType { @@ -37,7 +37,7 @@ public ClickHouseLancerDataType(ClickHouseDataType type) { } public ClickHouseLancerDataType(String textRepr) { - this.clickHouseType = ClickHouseDataType.fromTypeString(textRepr); + this.clickHouseType = ClickHouseDataType.of(textRepr); this.textRepr = textRepr; } @@ -59,8 +59,31 @@ public String toString() { public static class ClickHouseColumn extends AbstractTableColumn { - public ClickHouseColumn(String name, ClickHouseLancerDataType columnType) { - super(name, null, columnType); + private final boolean isAlias; + private final boolean isMaterialized; + + public ClickHouseColumn(String name, ClickHouseLancerDataType columnType, boolean isAlias, + boolean isMaterialized, ClickHouseTable table) { + super(name, table, columnType); + this.isAlias = isAlias; + this.isMaterialized = isMaterialized; + } + + public static ClickHouseSchema.ClickHouseColumn createDummy(String name, ClickHouseTable table) { + return new ClickHouseSchema.ClickHouseColumn(name, ClickHouseLancerDataType.getRandom(), false, false, + table); + } + + public boolean isAlias() { + return isAlias; + } + + public boolean isMaterialized() { + return isMaterialized; + } + + public ClickHouseColumnReference asColumnReference(String tableAlias) { + return new ClickHouseColumnReference(this, null, tableAlias); } } @@ -70,54 +93,77 @@ public static ClickHouseConstant getConstant(ResultSet randomRowValues, int colu Object value; ClickHouseConstant constant; if (randomRowValues.getString(columnIndex) == null) { - value = null; - constant = ClickHouseConstant.createNullConstant(); + constant = ClickHouseCreateConstant.createNullConstant(); } else { switch (valueType) { case Int32: value = randomRowValues.getLong(columnIndex); - constant = ClickHouseConstant.createInt32Constant((long) value); + constant = ClickHouseCreateConstant.createInt32Constant((long) value); break; case Float64: value = randomRowValues.getDouble(columnIndex); - constant = ClickHouseConstant.createFloat64Constant((double) value); + constant = ClickHouseCreateConstant.createFloat64Constant((double) value); break; case String: value = randomRowValues.getString(columnIndex); - constant = ClickHouseConstant.createStringConstant((String) value); + constant = ClickHouseCreateConstant.createStringConstant((String) value); break; + case AggregateFunction: + case Array: + // case Bool: + case Date: + // case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal128: + case Decimal256: case Decimal32: case Decimal64: - case Decimal128: - case Decimal: - case UUID: + // case Enum: + case Enum16: + case Enum8: case FixedString: - case Nothing: - case Nested: - case Tuple: + case Float32: + case IPv4: + case IPv6: + case Int128: case Int16: + case Int256: + case Int64: case Int8: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Float32: - case Array: - case AggregateFunction: - case Unknown: - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: case IntervalDay: case IntervalHour: + // case IntervalMicrosecond: + // case IntervalMillisecond: case IntervalMinute: + case IntervalMonth: + // case IntervalNanosecond: + case IntervalQuarter: case IntervalSecond: - case UInt64: - case UInt32: + case IntervalWeek: + case IntervalYear: + // case JSON: + // case LowCardinality: + case Map: + // case MultiPolygon: + case Nested: + case Nothing: + // case Nullable: + // case Object: + // case Point: + // case Polygon: + // case Ring: + // case SimpleAggregateFunction: + case Tuple: + case UInt128: case UInt16: + case UInt256: + case UInt32: + case UInt64: case UInt8: - case Int64: + case UUID: default: throw new AssertionError(valueType); } @@ -125,123 +171,20 @@ public static ClickHouseConstant getConstant(ResultSet randomRowValues, int colu return constant; } - public static class ClickHouseRowValue { - private final ClickHouseSchema.ClickHouseTables tables; - private final Map values; + public static class ClickHouseRowValue + extends AbstractRowValue { ClickHouseRowValue(ClickHouseSchema.ClickHouseTables tables, Map values) { - this.tables = tables; - this.values = values; - } - - public ClickHouseSchema.ClickHouseTables getTable() { - return tables; - } - - public Map getValues() { - return values; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - int i = 0; - for (ClickHouseSchema.ClickHouseColumn c : tables.getColumns()) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(values.get(c)); - } - return sb.toString(); - } - - public String getRowValuesAsString() { - List columnsToCheck = tables.getColumns(); - return getRowValuesAsString(columnsToCheck); - } - - public String getRowValuesAsString(List columnsToCheck) { - StringBuilder sb = new StringBuilder(); - Map expectedValues = getValues(); - for (int i = 0; i < columnsToCheck.size(); i++) { - if (i != 0) { - sb.append(", "); - } - ClickHouseConstant expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); - ClickHouseToStringVisitor visitor = new ClickHouseToStringVisitor(); - visitor.visit(expectedColumnValue); - sb.append(visitor.get()); - } - return sb.toString(); + super(tables, values); } } - public static class ClickHouseTables { - private final List tables; - private final List columns; + public static class ClickHouseTables extends AbstractTables { public ClickHouseTables(List tables) { - this.tables = tables; - columns = new ArrayList<>(); - for (ClickHouseSchema.ClickHouseTable t : tables) { - columns.addAll(t.getColumns()); - } - } - - public String tableNamesAsString() { - return tables.stream().map(t -> t.getName()).collect(Collectors.joining(", ")); - } - - public List getTables() { - return tables; - } - - public List getColumns() { - return columns; - } - - public String columnNamesAsString() { - return getColumns().stream().map(t -> t.getTable().getName() + "." + t.getName()) - .collect(Collectors.joining(", ")); - } - - public String columnNamesAsString(Function function) { - return getColumns().stream().map(function).collect(Collectors.joining(", ")); - } - - public ClickHouseRowValue getRandomRowValue(Connection con, StateToReproduce.ClickHouseStateToReproduce state) - throws SQLException { - String randomRow = String.format("SELECT %s, %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( - c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), - columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + c.getName() + ")"), - tableNamesAsString()); - Map values = new HashMap<>(); - try (Statement s = con.createStatement()) { - ResultSet randomRowValues; - try { - randomRowValues = s.executeQuery(randomRow); - } catch (SQLException e) { - throw new IgnoreMeException(); - } - if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow + "\n" + state); - } - for (int i = 0; i < getColumns().size(); i++) { - ClickHouseSchema.ClickHouseColumn column = getColumns().get(i); - int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); - assert columnIndex == i + 1; - String typeString = randomRowValues.getString(columnIndex + getColumns().size()); - ClickHouseDataType valueType = getColumnType(typeString).getType(); - ClickHouseConstant constant = getConstant(randomRowValues, columnIndex, valueType); - values.put(column, constant); - } - assert !randomRowValues.next(); - state.randomRowValues = values; - return new ClickHouseSchema.ClickHouseRowValue(this, values); - } - + super(tables); } } @@ -258,7 +201,8 @@ private static ClickHouseLancerDataType getColumnType(String typeString) { return new ClickHouseLancerDataType(typeString); } - public static class ClickHouseTable extends AbstractTable { + public static class ClickHouseTable + extends AbstractRelationalTable { public ClickHouseTable(String tableName, List columns, List indexes, boolean isView) { @@ -266,13 +210,13 @@ public ClickHouseTable(String tableName, List columns, List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); List indexes = Collections.emptyList(); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); ClickHouseTable t = new ClickHouseTable(tableName, databaseColumns, indexes, isView); for (ClickHouseColumn c : databaseColumns) { c.setTable(t); @@ -283,7 +227,7 @@ public static ClickHouseSchema fromConnection(Connection con, String databaseNam return new ClickHouseSchema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { ResultSet tableRs = s.executeQuery("SHOW TABLES"); @@ -295,14 +239,18 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("DESCRIBE " + tableName)) { while (rs.next()) { String columnName = rs.getString("name"); String dataType = rs.getString("type"); - ClickHouseColumn c = new ClickHouseColumn(columnName, getColumnType(dataType)); + String defaultType = rs.getString("default_type"); + boolean isAlias = "ALIAS".compareTo(defaultType) == 0; + boolean isMaterialized = "MATERIALIZED".compareTo(defaultType) == 0; + ClickHouseColumn c = new ClickHouseColumn(columnName, getColumnType(dataType), isAlias, + isMaterialized, null); columns.add(c); } } diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index ab966f7d9..29bcddcb7 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -1,7 +1,10 @@ package sqlancer.clickhouse; +import java.util.List; + import sqlancer.clickhouse.ast.ClickHouseAggregate; -import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; @@ -11,7 +14,7 @@ import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.visitor.ToStringVisitor; +import sqlancer.common.visitor.ToStringVisitor; public class ClickHouseToStringVisitor extends ToStringVisitor implements ClickHouseVisitor { @@ -20,17 +23,6 @@ public void visitSpecific(ClickHouseExpression expr) { ClickHouseVisitor.super.visit(expr); } - @Override - public void visit(ClickHouseBinaryComparisonOperation op) { - sb.append("("); - visit(op.getLeft()); - sb.append(") "); - sb.append(op.getOperator().getTextRepresentation()); - sb.append(" ("); - visit(op.getRight()); - sb.append(")"); - } - @Override public void visit(ClickHouseBinaryLogicalOperation op) { sb.append("("); @@ -70,9 +62,29 @@ public void visit(ClickHouseSelect select, boolean inner) { sb.append("("); } sb.append("SELECT "); + switch (select.getFromOptions()) { + case DISTINCT: + sb.append("DISTINCT "); + break; + case ALL: + sb.append(""); + break; + default: + throw new AssertionError(select.getFromOptions()); + } + visit(select.getFetchColumns()); - sb.append(" FROM "); - visit(select.getFromList()); + List fromList = select.getFromList(); + if (fromList != null) { + sb.append(" FROM "); + visit(fromList); + } + List joins = select.getJoinClauses(); + if (!joins.isEmpty()) { + for (ClickHouseExpression.ClickHouseJoin join : joins) { + visit(join); + } + } if (select.getWhereClause() != null) { sb.append(" WHERE "); visit(select.getWhereClause()); @@ -85,9 +97,9 @@ public void visit(ClickHouseSelect select, boolean inner) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByClause().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByClause()); + visit(select.getOrderByClauses()); } if (inner) { sb.append(")"); @@ -96,7 +108,12 @@ public void visit(ClickHouseSelect select, boolean inner) { @Override public void visit(ClickHouseTableReference tableReference) { - sb.append(tableReference.getTable().getName()); + sb.append(tableReference.getTable().getName()); // Original name, not alias. + String alias = tableReference.getAlias(); + if (alias != null) { + sb.append(" AS " + alias); + } + } @Override @@ -118,16 +135,70 @@ public void visit(ClickHouseCastOperation cast) { @Override public void visit(ClickHouseExpression.ClickHouseJoin join) { - + ClickHouseExpression.ClickHouseJoin.JoinType type = join.getType(); + if (type == ClickHouseExpression.ClickHouseJoin.JoinType.CROSS) { + sb.append(" JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.INNER) { + sb.append(" INNER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_OUTER) { + sb.append(" LEFT OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_OUTER) { + sb.append(" RIGHT OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.FULL_OUTER) { + sb.append(" FULL OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_ANTI) { + sb.append(" LEFT ANTI JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_ANTI) { + sb.append(" RIGHT ANTI JOIN "); + visit(join.getRightTable()); + } else { + throw new UnsupportedOperationException(); + } + ClickHouseExpression onClause = join.getOnClause(); + if (onClause != null) { + sb.append(" ON "); + visit(onClause); + } } @Override public void visit(ClickHouseColumnReference c) { - if (c.getColumn().getTable() == null) { + if (c.getTableAlias() != null) { + sb.append(c.getTableAlias()); + sb.append("."); + sb.append(c.getColumn().getName()); + } else if (c.getColumn().getTable() == null) { sb.append(c.getColumn().getName()); } else { sb.append(c.getColumn().getFullQualifiedName()); } + if (c.getAlias() != null) { + sb.append(" AS " + c.getAlias()); + } + } + + @Override + public void visit(ClickHouseBinaryFunctionOperation func) { + sb.append(func.getOperatorRepresentation()); + sb.append("("); + visit(func.getLeft()); + sb.append(","); + visit(func.getRight()); + sb.append(")"); + } + + @Override + public void visit(ClickHouseAliasOperation alias) { + visit(alias.getExpression()); + sb.append(" AS `"); + sb.append(alias.getAlias()); + sb.append("`"); } public static String asString(ClickHouseExpression expr) { diff --git a/src/sqlancer/clickhouse/ClickHouseVisitor.java b/src/sqlancer/clickhouse/ClickHouseVisitor.java index 25fa65466..2966f93be 100644 --- a/src/sqlancer/clickhouse/ClickHouseVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseVisitor.java @@ -1,7 +1,9 @@ package sqlancer.clickhouse; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; @@ -43,16 +45,26 @@ default void visit(ClickHouseColumnReference columnReference) { }; + default void visit(ClickHouseExpression.ClickHousePostfixText op) { + + } + void visit(ClickHouseTableReference tableReference); void visit(ClickHouseCastOperation cast); + void visit(ClickHouseAliasOperation alias); + void visit(ClickHouseExpression.ClickHouseJoin join); void visit(ClickHouseAggregate aggregate); + void visit(ClickHouseBinaryFunctionOperation func); + default void visit(ClickHouseExpression expr) { - if (expr instanceof ClickHouseBinaryComparisonOperation) { + if (expr instanceof ClickHouseBinaryFunctionOperation) { + visit((ClickHouseBinaryFunctionOperation) expr); + } else if (expr instanceof ClickHouseBinaryComparisonOperation) { visit((ClickHouseBinaryComparisonOperation) expr); } else if (expr instanceof ClickHouseBinaryLogicalOperation) { visit((ClickHouseBinaryLogicalOperation) expr); @@ -70,8 +82,14 @@ default void visit(ClickHouseExpression expr) { visit((ClickHouseCastOperation) expr); } else if (expr instanceof ClickHouseExpression.ClickHouseJoin) { visit((ClickHouseExpression.ClickHouseJoin) expr); + } else if (expr instanceof ClickHouseExpression.ClickHousePostfixText) { + visit((ClickHouseExpression.ClickHousePostfixText) expr); } else if (expr instanceof ClickHouseAggregate) { visit((ClickHouseAggregate) expr); + } else if (expr instanceof ClickHouseAliasOperation) { + visit((ClickHouseAliasOperation) expr); + } else if (expr instanceof ClickHouseExpression.ClickHouseJoinOnClause) { + visit((ClickHouseExpression.ClickHouseJoinOnClause) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java index 40803d782..a0483bd57 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java @@ -1,23 +1,23 @@ package sqlancer.clickhouse.ast; -import sqlancer.Randomly; -import sqlancer.clickhouse.ClickHouseSchema; -import ru.yandex.clickhouse.domain.ClickHouseDataType; - import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.Randomly; +import sqlancer.clickhouse.ClickHouseSchema; + public class ClickHouseAggregate extends ClickHouseExpression { private final ClickHouseAggregate.ClickHouseAggregateFunction func; - private final List expr; + private final ClickHouseExpression expr; public enum ClickHouseAggregateFunction { AVG(ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.Float32, ClickHouseDataType.Float64), - BOOL_AND(ClickHouseDataType.UInt8), BOOL_OR(ClickHouseDataType.UInt8), COUNT(ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.Float32, ClickHouseDataType.Float64, @@ -41,8 +41,8 @@ public static ClickHouseAggregateFunction getRandom(ClickHouseDataType type) { return Randomly.fromOptions(values()); } - public List getTypes(ClickHouseDataType returnType) { - return Arrays.asList(returnType); + public ClickHouseDataType getType(ClickHouseDataType returnType) { + return returnType; } public boolean supportsReturnType(ClickHouseDataType returnType) { @@ -65,7 +65,7 @@ public ClickHouseSchema.ClickHouseLancerDataType getRandomReturnType() { } - public ClickHouseAggregate(List expr, ClickHouseAggregateFunction func) { + public ClickHouseAggregate(ClickHouseExpression expr, ClickHouseAggregateFunction func) { this.expr = expr; this.func = func; } @@ -74,7 +74,7 @@ public ClickHouseAggregate.ClickHouseAggregateFunction getFunc() { return func; } - public List getExpr() { + public ClickHouseExpression getExpr() { return expr; } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java new file mode 100644 index 000000000..b7976ff46 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java @@ -0,0 +1,28 @@ +package sqlancer.clickhouse.ast; + +public class ClickHouseAliasOperation extends ClickHouseExpression { + + private final ClickHouseExpression expression; + private final String alias; + + public ClickHouseAliasOperation(ClickHouseExpression expression, String alias) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.alias = alias; + } + + @Override + public ClickHouseConstant getExpectedValue() { + return expression.getExpectedValue(); + } + + public ClickHouseExpression getExpression() { + return expression; + } + + public String getAlias() { + return alias; + } +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java new file mode 100644 index 000000000..fc0f8504a --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java @@ -0,0 +1,65 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; + +public class ClickHouseBinaryArithmeticOperation extends ClickHouseExpression + implements BinaryOperation { + + public enum ClickHouseBinaryArithmeticOperator { + ADD("+"), // + MINUS("-"), // + MULT("*"), // + DIV("/"), // + MODULO("%"); // + + String textRepresentation; + + ClickHouseBinaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ClickHouseBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + private final ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator operation; + private final ClickHouseExpression left; + private final ClickHouseExpression right; + + public ClickHouseBinaryArithmeticOperation(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator operation) { + this.left = left; + this.right = right; + this.operation = operation; + } + + public ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator getOperator() { + return operation; + } + + @Override + public ClickHouseExpression getLeft() { + return left; + } + + @Override + public ClickHouseExpression getRight() { + return right; + } + + @Override + public String getOperatorRepresentation() { + return operation.getTextRepresentation(); + } + + public static ClickHouseBinaryArithmeticOperation create(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator op) { + return new ClickHouseBinaryArithmeticOperation(left, right, op); + } +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java index 76115c6d1..75dc259e8 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java @@ -1,9 +1,11 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; -import sqlancer.visitor.BinaryOperation; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; +import sqlancer.common.visitor.BinaryOperation; public class ClickHouseBinaryComparisonOperation extends ClickHouseExpression implements BinaryOperation { @@ -75,7 +77,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { } else if (lessThan.asInt() >= 1) { return lessThan; } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } } @@ -95,7 +97,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { && equals.getDataType() == ClickHouseDataType.UInt32 && equals.getDataType() == ClickHouseDataType.Int64 && equals.getDataType() == ClickHouseDataType.UInt64 && equals.asInt() == 1) { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } else { ClickHouseConstant applyLess = left.applyLess(right); if (applyLess == null) { @@ -121,7 +123,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { && lessThan.getDataType() == ClickHouseDataType.UInt32 && lessThan.getDataType() == ClickHouseDataType.Int64 && lessThan.getDataType() == ClickHouseDataType.UInt64 && lessThan.asInt() >= 1) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else { ClickHouseConstant applyLess = left.applyLess(right); if (applyLess == null) { @@ -146,14 +148,14 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { ClickHouseConstant applyEquals = left.applyEquals(right); if (applyEquals == null) { return null; } boolean equals = applyEquals.asInt() == 1; - return ClickHouseConstant.createBoolean(!equals); + return ClickHouseCreateConstant.createBoolean(!equals); } } @@ -165,7 +167,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } ClickHouseConstant leftStr = ClickHouseCast.castToText(left); ClickHouseConstant rightStr = ClickHouseCast.castToText(right); @@ -173,7 +175,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } boolean val = LikeImplementationHelper.match(leftStr.asString(), rightStr.asString(), 0, 0, false); - return ClickHouseConstant.createBoolean(val); + return ClickHouseCreateConstant.createBoolean(val); } }; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java new file mode 100644 index 000000000..65bf4c79e --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java @@ -0,0 +1,57 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; + +public class ClickHouseBinaryFunctionOperation extends ClickHouseExpression { + + public enum ClickHouseBinaryFunctionOperator { + INT_DIV("intDiv"), GCD("gcd"), LCM("lcm"), MAX2("max2"), MIN2("min2"), POW("pow"); + + String textRepresentation; + + ClickHouseBinaryFunctionOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ClickHouseBinaryFunctionOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + private final ClickHouseBinaryFunctionOperator operation; + private final ClickHouseExpression left; + private final ClickHouseExpression right; + + public ClickHouseBinaryFunctionOperation(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryFunctionOperator operation) { + this.left = left; + this.right = right; + this.operation = operation; + } + + public ClickHouseBinaryFunctionOperator getOperator() { + return operation; + } + + public ClickHouseExpression getLeft() { + return left; + } + + public ClickHouseExpression getRight() { + return right; + } + + public String getOperatorRepresentation() { + return operation.getTextRepresentation(); + } + + public static ClickHouseBinaryFunctionOperation create(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryFunctionOperator op) { + return new ClickHouseBinaryFunctionOperation(left, right, op); + } + +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java index bbd37e865..019fad752 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.ast; import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; public class ClickHouseBinaryLogicalOperation extends ClickHouseExpression { @@ -14,22 +15,22 @@ public enum ClickHouseBinaryLogicalOperator { @Override public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { if (left.isNull() && right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else if (left.isNull()) { if (right.asBooleanNotNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } else if (right.isNull()) { if (left.asBooleanNotNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } else { - return left.asBooleanNotNull() && right.asBooleanNotNull() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); + return left.asBooleanNotNull() && right.asBooleanNotNull() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); } } }, @@ -37,13 +38,13 @@ public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant righ @Override public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { if (!left.isNull() && left.asBooleanNotNull()) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else if (!right.isNull() && right.asBooleanNotNull()) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } }; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCast.java b/src/sqlancer/clickhouse/ast/ClickHouseCast.java index 9fe2392cf..09c946717 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCast.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCast.java @@ -1,13 +1,15 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; - import java.math.BigDecimal; import java.sql.Connection; import java.sql.Statement; import java.util.Optional; import java.util.regex.Pattern; +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; + public final class ClickHouseCast extends ClickHouseExpression { private static final double MAX_INT_FOR_WHICH_CONVERSION_TO_INT_IS_TRIED = Math.pow(2, 51 - 1) - 1; @@ -50,18 +52,18 @@ public static Optional isTrue(ClickHouseConstant value) { public static ClickHouseConstant castToInt(ClickHouseConstant cons) { switch (cons.getDataType()) { case Nothing: - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); case Int32: return cons; case Float64: - return ClickHouseConstant.createInt32Constant((long) cons.asDouble()); + return ClickHouseCreateConstant.createInt32Constant((long) cons.asDouble()); case String: String asString = cons.asString(); while (startsWithWhitespace(asString)) { asString = asString.substring(1); } if (!asString.isEmpty() && unprintAbleCharThatLetsBecomeNumberZero(asString)) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } for (int i = asString.length(); i >= 0; i--) { try { @@ -79,13 +81,13 @@ public static ClickHouseConstant castToInt(ClickHouseConstant cons) { result = Long.MAX_VALUE; } } - return ClickHouseConstant.createInt32Constant(result); + return ClickHouseCreateConstant.createInt32Constant(result); } } catch (Exception e) { } } - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); default: throw new AssertionError(); } @@ -95,7 +97,7 @@ public static ClickHouseConstant castToInt(ClickHouseConstant cons) { public static ClickHouseConstant castToReal(ClickHouseConstant cons) { ClickHouseConstant numericValue = castToNumeric(cons); if (numericValue.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createFloat64Constant(numericValue.asInt()); + return ClickHouseCreateConstant.createFloat64Constant(numericValue.asInt()); } else { return numericValue; } @@ -109,7 +111,7 @@ public static ClickHouseConstant castToNumericFromNumOperand(ClickHouseConstant return convertInternal(value, false, false, false); } - /** + /* * Applies numeric affinity to a value. */ public static ClickHouseConstant castToNumeric(ClickHouseConstant value) { @@ -120,7 +122,7 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool boolean noNumIsRealZero, boolean convertIntToReal) throws AssertionError { switch (value.getDataType()) { case Nothing: - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); case Int32: case Float64: return value; @@ -130,16 +132,16 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool asString = asString.substring(1); } if (!asString.isEmpty() && unprintAbleCharThatLetsBecomeNumberZero(asString)) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } if (asString.toLowerCase().startsWith("-infinity") || asString.toLowerCase().startsWith("infinity") || asString.startsWith("NaN")) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } for (int i = asString.length(); i >= 0; i--) { try { String substring = asString.substring(0, i); - double d = Double.valueOf(substring); + double d = Double.parseDouble(substring); BigDecimal first = new BigDecimal(substring); long longValue = first.longValue(); BigDecimal second = BigDecimal.valueOf(longValue); @@ -151,17 +153,17 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool boolean isInteger = !isFloatingPointNumber && first.compareTo(second) == 0; if (doubleShouldBeConvertedToInt || isInteger && !convertIntToReal) { // see https://www.sqlite.org/src/tktview/afdc5a29dc - return ClickHouseConstant.createInt32Constant(first.longValue()); + return ClickHouseCreateConstant.createInt32Constant(first.longValue()); } else { - return ClickHouseConstant.createFloat64Constant(d); + return ClickHouseCreateConstant.createFloat64Constant(d); } } catch (Exception e) { } } if (noNumIsRealZero) { - return ClickHouseConstant.createFloat64Constant(0.0); + return ClickHouseCreateConstant.createFloat64Constant(0.0); } else { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } default: throw new AssertionError(value); @@ -222,14 +224,14 @@ public static ClickHouseConstant castToText(ClickHouseConstant cons) { } if (cons.getDataType() == ClickHouseDataType.Float64) { if (cons.asDouble() == Double.POSITIVE_INFINITY) { - return ClickHouseConstant.createStringConstant("Inf"); + return ClickHouseCreateConstant.createStringConstant("Inf"); } else if (cons.asDouble() == Double.NEGATIVE_INFINITY) { - return ClickHouseConstant.createStringConstant("-Inf"); + return ClickHouseCreateConstant.createStringConstant("-Inf"); } return castRealToText(cons); } if (cons.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createStringConstant(String.valueOf(cons.asInt())); + return ClickHouseCreateConstant.createStringConstant(String.valueOf(cons.asInt())); } return null; } @@ -237,7 +239,7 @@ public static ClickHouseConstant castToText(ClickHouseConstant cons) { private static synchronized ClickHouseConstant castRealToText(ClickHouseConstant cons) throws AssertionError { try (Statement s = castDatabase.createStatement()) { String castResult = s.executeQuery("SELECT CAST(" + cons.asDouble() + " AS TEXT)").getString(1); - return ClickHouseConstant.createStringConstant(castResult); + return ClickHouseCreateConstant.createStringConstant(castResult); } catch (Exception e) { throw new AssertionError(e); } @@ -246,9 +248,9 @@ private static synchronized ClickHouseConstant castRealToText(ClickHouseConstant public static ClickHouseConstant asBoolean(ClickHouseConstant val) { Optional boolVal = isTrue(val); if (boolVal.isPresent()) { - return ClickHouseConstant.createBoolean(boolVal.get()); + return ClickHouseCreateConstant.createBoolean(boolVal.get()); } else { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java index f47d2dca4..a905b4e49 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java @@ -1,6 +1,6 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java index 818da0e7e..6d75cf5fa 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java @@ -5,28 +5,30 @@ public class ClickHouseColumnReference extends ClickHouseExpression { private final ClickHouseColumn column; - private final ClickHouseConstant value; + private final String columnAlias; + private final String tableAlias; - public ClickHouseColumnReference(ClickHouseColumn column, ClickHouseConstant value) { + public ClickHouseColumnReference(ClickHouseColumn column, String columnAlias, String tableAlias) { this.column = column; - this.value = value; + this.columnAlias = columnAlias; + this.tableAlias = tableAlias; } - public static ClickHouseColumnReference create(ClickHouseColumn column, ClickHouseConstant value) { - return new ClickHouseColumnReference(column, value); + public ClickHouseColumnReference(ClickHouseAliasOperation alias) { + this.column = new ClickHouseColumn(alias.getAlias(), null, true, false, null); + this.columnAlias = null; + this.tableAlias = null; } public ClickHouseColumn getColumn() { return column; } - public ClickHouseConstant getValue() { - return value; + public String getAlias() { + return columnAlias; } - @Override - public ClickHouseConstant getExpectedValue() { - return value; + public String getTableAlias() { + return tableAlias; } - } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java index 54a3da2a9..b38ad7e89 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java @@ -1,1399 +1,13 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; -import sqlancer.IgnoreMeException; +import com.clickhouse.client.ClickHouseDataType; -import java.math.BigInteger; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; public abstract class ClickHouseConstant extends ClickHouseExpression { - public static class ClickHouseNullConstant extends ClickHouseConstant { - - @Override - public String toString() { - return "NULL"; - } - - @Override - public boolean isNull() { - return true; - } - - @Override - public boolean asBooleanNotNull() { - throw new AssertionError(); - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Nothing; - } - - @Override - public boolean compareInternal(Object value) { - return false; - } - - @Override - public ClickHouseConstant applyEquals(ClickHouseConstant right) { - return ClickHouseConstant.createNullConstant(); - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - return ClickHouseConstant.createNullConstant(); - } - - @Override - public Object getValue() { - return null; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - return null; - } - } - - public static class ClickHouseUInt8Constant extends ClickHouseConstant { - - private final int value; - - public ClickHouseUInt8Constant(int value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt8; - } - - @Override - public boolean compareInternal(Object val) { - return value == (int) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant((double) value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt8Constant extends ClickHouseConstant { - - private final int value; - - public ClickHouseInt8Constant(int value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int8; - } - - @Override - public boolean compareInternal(Object val) { - return value == (int) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt16Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseUInt16Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt16; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt16Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseInt16Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int16; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt32Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseUInt32Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt32; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt32Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseInt32Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int32; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt64Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseUInt64Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt64; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt64Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseInt64Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int64; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseFloat32Constant extends ClickHouseConstant { - - private final float value; - - public ClickHouseFloat32Constant(float value) { - this.value = value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public String toString() { - if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; - } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; - } - return String.valueOf(value); - } - - @Override - public boolean compareInternal(Object val) { - return Float.compare(value, (float) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asDouble() < right.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - ClickHouseConstant converted = right.cast(ClickHouseDataType.Float32); - return this.asDouble() < converted.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - - @Override - public boolean asBooleanNotNull() { - return Float.compare(value, (float) 0) == 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Float32; - } - - @Override - public double asDouble() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant((long) value); - case Int8: - return ClickHouseConstant.createInt8Constant((long) value); - case UInt16: - return ClickHouseConstant.createUInt16Constant((long) value); - case Int16: - return ClickHouseConstant.createInt16Constant((long) value); - case UInt32: - return ClickHouseConstant.createUInt32Constant((long) value); - case Int32: - return ClickHouseConstant.createInt32Constant((long) value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf((long) value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf((long) value)); - case Float32: - return ClickHouseConstant.createFloat32Constant(value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseFloat64Constant extends ClickHouseConstant { - - private final double value; - - public ClickHouseFloat64Constant(double value) { - this.value = value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public String toString() { - if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; - } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; - } - return String.valueOf(value); - } - - @Override - public boolean compareInternal(Object val) { - return Double.compare(value, (double) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asDouble() < right.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - ClickHouseConstant converted = right.cast(ClickHouseDataType.Float64); - return this.asDouble() < converted.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - - @Override - public boolean asBooleanNotNull() { - return Double.compare(value, 0.0) == 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Float64; - } - - @Override - public double asDouble() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant((long) value); - case Int8: - return ClickHouseConstant.createInt8Constant((long) value); - case UInt16: - return ClickHouseConstant.createUInt16Constant((long) value); - case Int16: - return ClickHouseConstant.createInt16Constant((long) value); - case UInt32: - return ClickHouseConstant.createUInt32Constant((long) value); - case Int32: - return ClickHouseConstant.createInt32Constant((long) value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf((long) value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf((long) value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseStringConstant extends ClickHouseConstant { - - private final String value; - - public ClickHouseStringConstant(String value) { - this.value = value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public String toString() { - return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; - } - - @Override - public boolean asBooleanNotNull() { - return value.length() > 0; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((String) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asString().compareTo(right.asString()) <= 0 ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.String; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(Integer.parseInt(value)); - case Int8: - return ClickHouseConstant.createInt8Constant(Integer.parseInt(value)); - case UInt16: - return ClickHouseConstant.createUInt16Constant(Integer.parseInt(value)); - case Int16: - return ClickHouseConstant.createInt16Constant(Integer.parseInt(value)); - case UInt32: - return ClickHouseConstant.createUInt32Constant(Integer.parseInt(value)); - case Int32: - return ClickHouseConstant.createInt32Constant(Integer.parseInt(value)); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) Float.parseFloat(value)); - case Float64: - return ClickHouseConstant.createFloat64Constant((double) Double.parseDouble(value)); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static ClickHouseConstant createStringConstant(String text) { - return new ClickHouseStringConstant(text); - } - - public static ClickHouseConstant createFloat64Constant(double val) { - return new ClickHouseFloat64Constant(val); - } - - public static ClickHouseConstant createFloat32Constant(float val) { - return new ClickHouseFloat32Constant(val); - } - - public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long val) { - switch (type) { - case IntervalYear: - break; - case IntervalQuarter: - break; - case IntervalMonth: - break; - case IntervalWeek: - break; - case IntervalDay: - break; - case IntervalHour: - break; - case IntervalMinute: - break; - case IntervalSecond: - break; - case UInt64: - return createUInt64Constant(BigInteger.valueOf(val)); - case UInt32: - return createUInt32Constant(val); - case UInt16: - return createUInt16Constant(val); - case UInt8: - return createUInt8Constant(val); - case Int64: - return createInt64Constant(BigInteger.valueOf(val)); - case Int32: - return createInt32Constant(val); - case Int16: - return createInt16Constant(val); - case Int8: - return createInt8Constant(val); - case Date: - break; - case DateTime: - break; - case Enum8: - break; - case Enum16: - break; - case Float32: - break; - case Float64: - break; - case Decimal32: - break; - case Decimal64: - break; - case Decimal128: - break; - case Decimal: - break; - case UUID: - break; - case String: - break; - case FixedString: - break; - case Nothing: - break; - case Nested: - break; - case Tuple: - break; - case Array: - break; - case AggregateFunction: - break; - case Unknown: - break; - default: - break; - } - throw new AssertionError(type); - } - - public static ClickHouseConstant createInt64Constant(BigInteger val) { - return new ClickHouseInt64Constant(val); - } - - public static ClickHouseConstant createUInt64Constant(BigInteger val) { - return new ClickHouseUInt64Constant(val); - } - - public static ClickHouseConstant createInt32Constant(long val) { - return new ClickHouseInt32Constant(val); - } - - public static ClickHouseConstant createUInt32Constant(long val) { - return new ClickHouseUInt32Constant(val); - } - - public static ClickHouseConstant createUInt16Constant(long val) { - return new ClickHouseUInt16Constant(val); - } - - public static ClickHouseConstant createInt16Constant(long val) { - return new ClickHouseInt16Constant(val); - } - - public static ClickHouseConstant createUInt8Constant(long val) { - return new ClickHouseUInt8Constant((int) val); - } - - public static ClickHouseConstant createInt8Constant(long val) { - return new ClickHouseInt8Constant((int) val); - } - public abstract boolean isNull(); - public static ClickHouseConstant createNullConstant() { - return new ClickHouseNullConstant(); - } - - public static ClickHouseConstant createTrue() { - return new ClickHouseUInt8Constant(1); - } - - public static ClickHouseConstant createFalse() { - return new ClickHouseUInt8Constant(0); - } - - public static ClickHouseConstant createBoolean(boolean val) { - return val ? createTrue() : createFalse(); - } - public abstract ClickHouseConstant cast(ClickHouseDataType type); public abstract boolean asBooleanNotNull(); @@ -1404,8 +18,8 @@ public static ClickHouseConstant createBoolean(boolean val) { public ClickHouseConstant applyEquals(ClickHouseConstant right) { if (this.getDataType() == right.getDataType()) { - return this.compareInternal(right.getValue()) ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); + return this.compareInternal(right.getValue()) ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); } else { ClickHouseConstant converted = right.cast(this.getDataType()); return this.applyEquals(converted); diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index 16356c9cd..eff88d012 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -1,8 +1,13 @@ package sqlancer.clickhouse.ast; -import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.UnaryOperation; -public abstract class ClickHouseExpression { +public abstract class ClickHouseExpression implements Expression { public ClickHouseConstant getExpectedValue() { return null; @@ -34,35 +39,71 @@ public ClickHouseExpression getExpression() { } } - public static class ClickHouseJoin extends ClickHouseExpression { + public static class ClickHouseJoinOnClause extends ClickHouseExpression + implements BinaryOperation { + private final ClickHouseExpression left; + private final ClickHouseExpression right; + public ClickHouseJoinOnClause(ClickHouseExpression left, ClickHouseExpression right) { + this.left = left; + this.right = right; + } + + @Override + public final ClickHouseExpression getLeft() { + return this.left; + } + + @Override + public final ClickHouseExpression getRight() { + return this.right; + } + + @Override + public String getOperatorRepresentation() { + return "="; + } + } + + public static class ClickHouseJoin extends ClickHouseExpression + implements Join { // TODO: support ANY, ALL, ASOF modifiers + // LEFT_SEMI, RIGHT_SEMI are not deterministic as ClickHouse allows to read columns from + // whitelist table as well public enum JoinType { - INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, NATURAL, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; + INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_ANTI, RIGHT_ANTI; } - private final ClickHouseSchema.ClickHouseTable table; + private final ClickHouseTableReference leftTable; + private final ClickHouseTableReference rightTable; private ClickHouseExpression onClause; private final ClickHouseJoin.JoinType type; - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable table, ClickHouseExpression onClause, - ClickHouseJoin.JoinType type) { - this.table = table; + public ClickHouseJoin(ClickHouseTableReference leftTable, ClickHouseTableReference rightTable, + ClickHouseJoin.JoinType type, ClickHouseJoinOnClause onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; this.onClause = onClause; this.type = type; } - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable table, ClickHouseJoin.JoinType type) { - this.table = table; - if (type != ClickHouseJoin.JoinType.NATURAL) { + public ClickHouseJoin(ClickHouseTableReference leftTable, ClickHouseTableReference rightTable, + ClickHouseJoin.JoinType type) { + this.leftTable = leftTable; + this.rightTable = rightTable; + if (type != ClickHouseJoin.JoinType.CROSS) { throw new AssertionError(); } this.onClause = null; this.type = type; } - public ClickHouseSchema.ClickHouseTable getTable() { - return table; + public ClickHouseTableReference getLeftTable() { + return leftTable; + } + + public ClickHouseTableReference getRightTable() { + return rightTable; } public ClickHouseExpression getOnClause() { @@ -73,6 +114,7 @@ public ClickHouseJoin.JoinType getType() { return type; } + @Override public void setOnClause(ClickHouseExpression onClause) { this.onClause = onClause; } @@ -96,4 +138,50 @@ public String getQuery() { } } + public static class ClickHousePostfixText extends ClickHouseExpression + implements UnaryOperation { + + private final ClickHouseExpression expr; + private final String text; + private ClickHouseConstant expectedValue; + + public ClickHousePostfixText(ClickHouseExpression expr, String text, ClickHouseConstant expectedValue) { + this.expr = expr; + this.text = text; + this.expectedValue = expectedValue; + } + + public ClickHousePostfixText(String text, ClickHouseConstant expectedValue) { + this(null, text, expectedValue); + } + + public String getText() { + return text; + } + + @Override + public ClickHouseConstant getExpectedValue() { + return expectedValue; + } + + @Override + public ClickHouseExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return getText(); + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java index 7a72f2919..61aeffa4d 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java @@ -3,10 +3,16 @@ import java.util.Collections; import java.util.List; -public class ClickHouseSelect extends ClickHouseExpression { +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ClickHouseToStringVisitor; +import sqlancer.common.ast.newast.Select; + +public class ClickHouseSelect extends ClickHouseExpression implements + Select { private ClickHouseSelect.SelectType fromOptions = ClickHouseSelect.SelectType.ALL; - private List fromList = Collections.emptyList(); + private List fromClauses; private ClickHouseExpression whereClause; private List groupByClause = Collections.emptyList(); private ClickHouseExpression limitClause; @@ -24,8 +30,13 @@ public void setSelectType(ClickHouseSelect.SelectType fromOptions) { this.setFromOptions(fromOptions); } - public void setFromTables(List fromTables) { - this.setFromList(fromTables); + public void setFromClause(ClickHouseExpression fromList) { + this.fromClauses = List.of(fromList); + } + + @Override + public List getFromList() { + return fromClauses; } public ClickHouseSelect.SelectType getFromOptions() { @@ -36,76 +47,94 @@ public void setFromOptions(ClickHouseSelect.SelectType fromOptions) { this.fromOptions = fromOptions; } - public List getFromList() { - return fromList; - } - - public void setFromList(List fromList) { - this.fromList = fromList; - } - + @Override public ClickHouseExpression getWhereClause() { return whereClause; } + @Override public void setWhereClause(ClickHouseExpression whereClause) { this.whereClause = whereClause; } + @Override public void setGroupByClause(List groupByClause) { this.groupByClause = groupByClause; } + @Override public List getGroupByClause() { return groupByClause; } + @Override public void setLimitClause(ClickHouseExpression limitClause) { this.limitClause = limitClause; } + @Override public ClickHouseExpression getLimitClause() { return limitClause; } - public List getOrderByClause() { + @Override + public List getOrderByClauses() { return orderByClause; } - public void setOrderByExpressions(List orderBy) { + @Override + public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } + @Override public void setOffsetClause(ClickHouseExpression offsetClause) { this.offsetClause = offsetClause; } + @Override public ClickHouseExpression getOffsetClause() { return offsetClause; } + @Override public void setFetchColumns(List fetchColumns) { this.fetchColumns = fetchColumns; } + @Override public List getFetchColumns() { return fetchColumns; } + @Override public void setJoinClauses(List joinStatements) { this.joinStatements = joinStatements; } + @Override public List getJoinClauses() { return joinStatements; } + @Override public void setHavingClause(ClickHouseExpression havingClause) { this.havingClause = havingClause; } + @Override public ClickHouseExpression getHavingClause() { assert orderByClause != null; return havingClause; } + + @Override + public String asString() { + return ClickHouseToStringVisitor.asString(this); + } + + @Override + public void setFromList(List fromList) { + this.fromClauses = fromList; + } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java b/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java index 415dfca22..406ffb15c 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java @@ -1,17 +1,34 @@ package sqlancer.clickhouse.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; public class ClickHouseTableReference extends ClickHouseExpression { private final ClickHouseTable table; + private final String alias; - public ClickHouseTableReference(ClickHouseTable table) { + public ClickHouseTableReference(ClickHouseTable table, String alias) { this.table = table; + this.alias = alias; } public ClickHouseTable getTable() { return table; } + public String getTableName() { + return (alias == null) ? table.getName() : alias; + } + + public String getAlias() { + return alias; + } + + public List getColumnReferences() { + return this.table.getColumns().stream().map(c -> c.asColumnReference(this.alias)).collect(Collectors.toList()); + } + } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java new file mode 100644 index 000000000..30707354f --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java @@ -0,0 +1,52 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.visitor.UnaryOperation; + +public class ClickHouseUnaryFunctionOperation extends ClickHouseExpression + implements UnaryOperation { + private final ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator operator; + private final ClickHouseExpression expression; + + public ClickHouseUnaryFunctionOperation(ClickHouseExpression expression, ClickHouseUnaryFunctionOperator operator) { + this.operator = operator; + this.expression = expression; + } + + public enum ClickHouseUnaryFunctionOperator implements Operator { + EXP("exp"), SQRT("sqrt"), ERF("erf"), SIN("sin"), COS("cos"), TAN("tan"), SIGN("sign"), RADIANS("radians"), + LOG("log"), ABS("abs"); + + private String textRepresentation; + + ClickHouseUnaryFunctionOperator(String text) { + this.textRepresentation = text; + } + + public static ClickHouseUnaryFunctionOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + @Override + public ClickHouseExpression getExpression() { + return expression; + } + + @Override + public String getOperatorRepresentation() { + return operator.getTextRepresentation(); + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.PREFIX; + } + +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java index 490980a02..444818413 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java @@ -1,8 +1,9 @@ package sqlancer.clickhouse.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.visitor.UnaryOperation; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.visitor.UnaryOperation; public class ClickHouseUnaryPostfixOperation extends ClickHouseExpression implements UnaryOperation { @@ -86,7 +87,7 @@ public ClickHouseConstant getExpectedValue() { if (negate) { val = !val; } - return ClickHouseConstant.createInt32Constant(val ? 1 : 0); + return ClickHouseCreateConstant.createInt32Constant(val ? 1 : 0); } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java index 4f90b0760..15ab8fb36 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java @@ -1,9 +1,11 @@ package sqlancer.clickhouse.ast; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.visitor.UnaryOperation; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.visitor.UnaryOperation; public class ClickHouseUnaryPrefixOperation extends ClickHouseExpression implements UnaryOperation { @@ -20,10 +22,10 @@ public enum ClickHouseUnaryPrefixOperator implements Operator { @Override public ClickHouseConstant apply(ClickHouseConstant constant) { if (constant.getDataType() == ClickHouseDataType.Nothing) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return constant.asBooleanNotNull() ? ClickHouseConstant.createFalse() - : ClickHouseConstant.createTrue(); + return constant.asBooleanNotNull() ? ClickHouseCreateConstant.createFalse() + : ClickHouseCreateConstant.createTrue(); } } }, @@ -31,7 +33,7 @@ public ClickHouseConstant apply(ClickHouseConstant constant) { @Override public ClickHouseConstant apply(ClickHouseConstant constant) { if (constant.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createInt32Constant(-constant.asInt()); + return ClickHouseCreateConstant.createInt32Constant(-constant.asInt()); } throw new AssertionError(constant); } diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java new file mode 100644 index 000000000..d5164162c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java @@ -0,0 +1,142 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseBooleanConstant extends ClickHouseConstant { + + private final boolean value; + + public ClickHouseBooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return value ? "true" : "false"; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Bool; + } + + @Override + public boolean compareInternal(Object val) { + if (val instanceof Boolean) { + return value == ((Boolean) val).booleanValue(); + } else { + return value == ((int) val != 0); + } + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value ? 1 : 0; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(asInt()); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(asInt()); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(asInt()); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(asInt()); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(asInt()); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(asInt()); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(asInt())); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(asInt())); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(asInt())); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(asInt())); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(asInt())); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(asInt())); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) asInt()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) asInt()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(asInt() != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java new file mode 100644 index 000000000..dc87619a8 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java @@ -0,0 +1,162 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; +import sqlancer.clickhouse.ast.ClickHouseExpression; + +public abstract class ClickHouseCreateConstant extends ClickHouseExpression { + + public static ClickHouseConstant createStringConstant(String text) { + return new ClickHouseStringConstant(text); + } + + public static ClickHouseConstant createFloat64Constant(double val) { + return new ClickHouseFloat64Constant(val); + } + + public static ClickHouseConstant createFloat32Constant(float val) { + return new ClickHouseFloat32Constant(val); + } + + public static ClickHouseConstant createInt256Constant(BigInteger val) { + return new ClickHouseInt256Constant(val); + } + + public static ClickHouseConstant createUInt256Constant(BigInteger val) { + return new ClickHouseUInt256Constant(val); + } + + public static ClickHouseConstant createInt128Constant(BigInteger val) { + return new ClickHouseInt128Constant(val); + } + + public static ClickHouseConstant createUInt128Constant(BigInteger val) { + return new ClickHouseUInt128Constant(val); + } + + public static ClickHouseConstant createInt64Constant(BigInteger val) { + return new ClickHouseInt64Constant(val); + } + + public static ClickHouseConstant createUInt64Constant(BigInteger val) { + return new ClickHouseUInt64Constant(val); + } + + public static ClickHouseConstant createInt32Constant(long val) { + return new ClickHouseInt32Constant(val); + } + + public static ClickHouseConstant createUInt32Constant(long val) { + return new ClickHouseUInt32Constant(val); + } + + public static ClickHouseConstant createUInt16Constant(long val) { + return new ClickHouseUInt16Constant(val); + } + + public static ClickHouseConstant createInt16Constant(long val) { + return new ClickHouseInt16Constant(val); + } + + public static ClickHouseConstant createUInt8Constant(long val) { + return new ClickHouseUInt8Constant((int) val); + } + + public static ClickHouseConstant createInt8Constant(long val) { + return new ClickHouseInt8Constant((int) val); + } + + public static ClickHouseConstant createBooleanConstant(Boolean b) { + return new ClickHouseBooleanConstant(b); + } + + public static ClickHouseConstant createNullConstant() { + return new ClickHouseNullConstant(); + } + + public static ClickHouseConstant createTrue() { + return new ClickHouseBooleanConstant(true); + } + + public static ClickHouseConstant createFalse() { + return new ClickHouseBooleanConstant(false); + } + + public static ClickHouseConstant createBoolean(boolean val) { + return val ? createTrue() : createFalse(); + } + + public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long val) { + switch (type) { + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(val)); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(val)); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(val)); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(val)); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(val)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(val)); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + case String: + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } + +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java new file mode 100644 index 000000000..1385790cb --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java @@ -0,0 +1,136 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseFloat32Constant extends ClickHouseConstant { + + private final float value; + + public ClickHouseFloat32Constant(float value) { + this.value = value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + @Override + public boolean compareInternal(Object val) { + return Float.compare(value, (float) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asDouble() < right.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + ClickHouseConstant converted = right.cast(ClickHouseDataType.Float32); + return this.asDouble() < converted.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + + @Override + public boolean asBooleanNotNull() { + return Float.compare(value, (float) 0) == 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Float32; + } + + @Override + public double asDouble() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant((long) value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant((long) value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant((long) value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant((long) value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant((long) value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant((long) value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf((long) value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf((long) value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java new file mode 100644 index 000000000..9146faa39 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java @@ -0,0 +1,136 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseFloat64Constant extends ClickHouseConstant { + + private final double value; + + public ClickHouseFloat64Constant(double value) { + this.value = value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + @Override + public boolean compareInternal(Object val) { + return Double.compare(value, (double) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asDouble() < right.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + ClickHouseConstant converted = right.cast(ClickHouseDataType.Float64); + return this.asDouble() < converted.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + + @Override + public boolean asBooleanNotNull() { + return Double.compare(value, 0.0) == 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Float64; + } + + @Override + public double asDouble() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant((long) value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant((long) value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant((long) value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant((long) value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant((long) value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant((long) value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf((long) value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf((long) value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java new file mode 100644 index 000000000..c388cb638 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt128Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt128Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int128; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java new file mode 100644 index 000000000..1fde4dfb1 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt16Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseInt16Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int16; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java new file mode 100644 index 000000000..f40ea5297 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt256Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt256Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int256; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java new file mode 100644 index 000000000..408df18fc --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt32Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseInt32Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int32; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java new file mode 100644 index 000000000..cb5c4b5f8 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java @@ -0,0 +1,131 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt64Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt64Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int64; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java new file mode 100644 index 000000000..38ad4878c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt8Constant extends ClickHouseConstant { + + private final int value; + + public ClickHouseInt8Constant(int value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int8; + } + + @Override + public boolean compareInternal(Object val) { + return value == (int) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java new file mode 100644 index 000000000..08370d560 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java @@ -0,0 +1,53 @@ +package sqlancer.clickhouse.ast.constant; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseNullConstant extends ClickHouseConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public boolean asBooleanNotNull() { + throw new AssertionError(); + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Nothing; + } + + @Override + public boolean compareInternal(Object value) { + return false; + } + + @Override + public ClickHouseConstant applyEquals(ClickHouseConstant right) { + return ClickHouseCreateConstant.createNullConstant(); + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + return ClickHouseCreateConstant.createNullConstant(); + } + + @Override + public Object getValue() { + return null; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + return null; + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java new file mode 100644 index 000000000..fa8ba11f5 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java @@ -0,0 +1,125 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseStringConstant extends ClickHouseConstant { + + private final String value; + + public ClickHouseStringConstant(String value) { + this.value = value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; + } + + @Override + public boolean asBooleanNotNull() { + return !value.isEmpty(); + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((String) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asString().compareTo(right.asString()) <= 0 ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.String; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(Integer.parseInt(value)); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(Integer.parseInt(value)); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(Integer.parseInt(value)); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(Integer.parseInt(value)); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(Integer.parseInt(value)); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(Integer.parseInt(value)); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) Float.parseFloat(value)); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) Double.parseDouble(value)); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value == "true"); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java new file mode 100644 index 000000000..5952c967c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt128Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt128Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt128; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java new file mode 100644 index 000000000..c7ae84fdf --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt16Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseUInt16Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt16; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java new file mode 100644 index 000000000..95b333323 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt256Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt256Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt256; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java new file mode 100644 index 000000000..8e4f5fd09 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt32Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseUInt32Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt32; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java new file mode 100644 index 000000000..cd3363850 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java @@ -0,0 +1,131 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt64Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt64Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt64; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java new file mode 100644 index 000000000..de0d308c3 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java @@ -0,0 +1,138 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt8Constant extends ClickHouseConstant { + + private final int value; + + public ClickHouseUInt8Constant(int value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt8; + } + + @Override + public boolean compareInternal(Object val) { + return value == (int) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(value)); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(value)); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(value)); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java new file mode 100644 index 000000000..2584ff0b3 --- /dev/null +++ b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java @@ -0,0 +1,99 @@ +package sqlancer.clickhouse.gen; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; + +public class ClickHouseColumnBuilder { + + private final StringBuilder sb = new StringBuilder(); + + private static boolean allowAlias = true; + private static boolean allowMaterialized = true; + private static boolean allowDefaultValue = true; + private static boolean allowCodec = true; + + private enum Constraints { + DEFAULT, MATERIALIZED, CODEC, ALIAS // TTL + } + + public String createColumn(String columnName, ClickHouseProvider.ClickHouseGlobalState globalState, + List columns) { + sb.append(columnName); + sb.append(" "); + List constraints = new ArrayList<>(); + ClickHouseSchema.ClickHouseLancerDataType dataType = ClickHouseSchema.ClickHouseLancerDataType.getRandom(); + if (Randomly.getBooleanWithSmallProbability()) { + constraints = Randomly.subset(Constraints.values()); + if (!allowAlias || columns.isEmpty() || columns.size() == 1) { + constraints.remove(Constraints.ALIAS); + } + if (!allowMaterialized) { + constraints.remove(Constraints.MATERIALIZED); + } + if (!allowDefaultValue) { + constraints.remove(Constraints.DEFAULT); + } + if (constraints.contains(Constraints.MATERIALIZED)) { + constraints.remove(Constraints.ALIAS); + constraints.remove(Constraints.DEFAULT); + } else if (constraints.contains(Constraints.ALIAS)) { + constraints.remove(Constraints.DEFAULT); + constraints.remove(Constraints.CODEC); + } + } + + if (!constraints.contains(Constraints.ALIAS)) { + sb.append(dataType); + } + + Collections.sort(constraints); + + for (Constraints c : constraints) { + switch (c) { + case MATERIALIZED: + if (allowMaterialized) { + sb.append(" MATERIALIZED ("); + sb.append( + ClickHouseVisitor.asString( + new ClickHouseExpressionGenerator(globalState).generateExpressionWithColumns( + columns.stream().filter(p -> !p.getName().contentEquals(columnName)) + .map(p -> p.asColumnReference(null)).collect(Collectors.toList()), + 2))); + sb.append(")"); + } + break; + case DEFAULT: + if (allowDefaultValue) { + sb.append(" DEFAULT "); + sb.append(new ClickHouseExpressionGenerator(globalState).generateConstant(dataType)); + } + break; + case ALIAS: + if (allowAlias) { + sb.append(" ALIAS "); + sb.append(Randomly.fromList(columns.stream().filter(p -> !p.getName().contentEquals(columnName)) + .collect(Collectors.toList())).getName()); + } + break; + case CODEC: + if (allowCodec) { + sb.append(" CODEC ("); + sb.append(Randomly.fromOptions("NONE", "ZSTD", "LZ4HC")); + sb.append(")"); + } + break; + default: + throw new AssertionError(); + } + } + return sb.toString(); + } + +} diff --git a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java index 467f8591c..deae2a9aa 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java @@ -1,12 +1,12 @@ package sqlancer.clickhouse.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseTableReference; -import java.util.ArrayList; -import java.util.List; - public final class ClickHouseCommon { private ClickHouseCommon() { @@ -16,12 +16,20 @@ public static String createColumnName(int nr) { return String.format("c%d", nr); } + public static String createTableName(int nr) { + return String.format("t%d", nr); + } + + public static String createConstraintName(int nr) { + return String.format("x%d", nr); + } + public static List getTableRefs(List tables, ClickHouseSchema s) { List tableRefs = new ArrayList<>(); for (ClickHouseSchema.ClickHouseTable t : tables) { ClickHouseTableReference tableRef; - tableRef = new ClickHouseTableReference(t); + tableRef = new ClickHouseTableReference(t, null); tableRefs.add(tableRef); } return tableRefs; diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 5cabcf358..4e04e91f4 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -1,76 +1,262 @@ package sqlancer.clickhouse.gen; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; - +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAggregate.ClickHouseAggregateFunction; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryArithmeticOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; -import sqlancer.clickhouse.ast.ClickHouseConstant; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; +import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.ast.ClickHouseTableReference; +import sqlancer.clickhouse.ast.ClickHouseUnaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; +import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.gen.TypedExpressionGenerator; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; +import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; public class ClickHouseExpressionGenerator - extends TypedExpressionGenerator { + extends TypedExpressionGenerator implements + NoRECGenerator, + TLPWhereGenerator { private final ClickHouseGlobalState globalState; public boolean allowAggregateFunctions; + private List tables; + private final List columnRefs; + public ClickHouseExpressionGenerator(ClickHouseGlobalState globalState) { this.globalState = globalState; + this.columnRefs = new ArrayList<>(); + } + + public final void addColumns(List col) { + this.columnRefs.addAll(col); + } + + private enum ColumnLike { + UNARY_PREFIX, BINARY_ARITHMETIC, UNARY_FUNCTION, BINARY_FUNCTION } private enum Expression { - UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL + UNARY_PREFIX, BINARY_ARITHMETIC, UNARY_FUNCTION, BINARY_FUNCTION, BINARY_LOGICAL, BINARY_COMPARISON, + UNARY_POSTFIX + } + + public ClickHouseExpression generateExpressionWithColumns(List columns, + int remainingDepth) { + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(null); + } + + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + return columns.get((int) Randomly.getNotCachedInteger(0, columns.size() - 1)); + } + + ColumnLike expr = Randomly.fromOptions(ColumnLike.values()); + switch (expr) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseUnaryPrefixOperator.MINUS); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(expr); + } + } + + public ClickHouseExpression generateAggregateExpressionWithColumns(List columns, + int remainingDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return new ClickHouseAggregate(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseAggregate.ClickHouseAggregateFunction.getRandom()); + } + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(null); + } + + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + return columns.get((int) Randomly.getNotCachedInteger(0, columns.size() - 1)); + } + + ColumnLike expr = Randomly.fromOptions(ColumnLike.values()); + switch (expr) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseUnaryPrefixOperator.MINUS); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(expr); + } + } + + public ClickHouseExpression generateExpressionWithExpression(List expression, + int remainingDepth) { + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + if (Randomly.getBoolean()) { + return expression.get((int) Randomly.getNotCachedInteger(0, expression.size() - 1)); + } else { + return generateConstant(null); + } + } + + Expression type = Randomly.fromOptions(Expression.values()); + switch (type) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new ClickHouseUnaryPostfixOperation(generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.getRandom(), false); + case BINARY_COMPARISON: + return new ClickHouseBinaryComparisonOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.getRandomOperator()); + case BINARY_LOGICAL: + return new ClickHouseBinaryLogicalOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryLogicalOperation.ClickHouseBinaryLogicalOperator.getRandom()); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(type); + } } @Override protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, int depth) { - if (allowAggregateFunctions && Randomly.getBoolean()) { - return generateAggregate(); + if (allowAggregateFunctions && Randomly.getBooleanWithRatherLowProbability()) { + ClickHouseLancerDataType aggType = ClickHouseLancerDataType.getRandom(); + return new ClickHouseAggregate(generateExpression(aggType, depth + 1), + ClickHouseAggregate.ClickHouseAggregateFunction.getRandom()); } - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { return generateLeafNode(type); } Expression expr = Randomly.fromOptions(Expression.values()); + ClickHouseLancerDataType leftLeafType = ClickHouseLancerDataType.getRandom(); + ClickHouseLancerDataType rightLeafType = ClickHouseLancerDataType.getRandom(); + if (Randomly.getBooleanWithRatherLowProbability()) { + rightLeafType = leftLeafType; + } + switch (expr) { case UNARY_PREFIX: - return new ClickHouseUnaryPrefixOperation(generateExpression(type, depth + 1), + return new ClickHouseUnaryPrefixOperation(generateExpression(leftLeafType, depth + 1), ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.getRandom()); case UNARY_POSTFIX: - return new ClickHouseUnaryPostfixOperation(generateExpression(type, depth + 1), + return new ClickHouseUnaryPostfixOperation(generateExpression(leftLeafType, depth + 1), ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.getRandom(), false); case BINARY_COMPARISON: - return new ClickHouseBinaryComparisonOperation(generateExpression(type, depth + 1), - generateExpression(type, depth + 1), + return new ClickHouseBinaryComparisonOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(rightLeafType, depth + 1), ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.getRandomOperator()); case BINARY_LOGICAL: - return new ClickHouseBinaryLogicalOperation(generateExpression(type, depth + 1), - generateExpression(type, depth + 1), + return new ClickHouseBinaryLogicalOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(rightLeafType, depth + 1), ClickHouseBinaryLogicalOperation.ClickHouseBinaryLogicalOperator.getRandom()); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(leftLeafType, depth + 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpression(leftLeafType, depth + 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(leftLeafType, depth + 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); default: throw new AssertionError(expr); } } + protected ClickHouseExpression.ClickHouseJoinOnClause generateJoinClause(ClickHouseTableReference leftTable, + ClickHouseTableReference rightTable) { + List leftColumns = leftTable.getColumnReferences(); + List rightColumns = rightTable.getColumnReferences(); + ClickHouseExpression leftExpr = generateExpressionWithColumns(leftColumns, 2); + ClickHouseExpression rightExpr = generateExpressionWithColumns(rightColumns, 2); + return new ClickHouseExpression.ClickHouseJoinOnClause(leftExpr, rightExpr); + } + @Override protected ClickHouseExpression generateColumn(ClickHouseLancerDataType type) { - List filteredColumns = columns.stream() - .filter(c -> c.getType().getType().name().equals(type.getType().name())).collect(Collectors.toList()); - ClickHouseColumn column = filteredColumns.isEmpty() ? Randomly.fromList(columns) - : Randomly.fromList(filteredColumns); - return new ClickHouseColumnReference(column, null); + if (columnRefs.isEmpty()) { + return generateConstant(type); + } + List filteredColumns = columnRefs.stream() + .filter(c -> c.getColumn().getType().getType().name().equals(type.getType().name())) + .collect(Collectors.toList()); + return filteredColumns.isEmpty() ? Randomly.fromList(columnRefs) : Randomly.fromList(filteredColumns); + } + + protected ClickHouseExpression getColumnNameFromTable(ClickHouseSchema.ClickHouseTable table) { + if (columnRefs.isEmpty()) { + return generateConstant(ClickHouseLancerDataType.getRandom()); + } + List filteredColumns = columnRefs.stream() + .filter(c -> c.getColumn().getTable() == table).collect(Collectors.toList()); + if (filteredColumns.isEmpty()) { + return generateConstant(ClickHouseLancerDataType.getRandom()); + } + return Randomly.fromList(filteredColumns); } @Override @@ -78,29 +264,29 @@ protected ClickHouseLancerDataType getRandomType() { return ClickHouseLancerDataType.getRandom(); } - public List getRandomJoinClauses( + public List getRandomJoinClauses(ClickHouseTableReference left, List tables) { List joinStatements = new ArrayList<>(); - if (!globalState.getDmbsSpecificOptions().testJoins) { + if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } - if (Randomly.getBoolean() && tables.size() > 1) { + List leftTables = new ArrayList<>(); + leftTables.add(left); + if (Randomly.getBoolean() && !tables.isEmpty()) { int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); for (int i = 0; i < nrJoinClauses; i++) { - ClickHouseExpression joinClause = generateExpression(ClickHouseLancerDataType.getRandom()); - ClickHouseSchema.ClickHouseTable table = Randomly.fromList(tables); - tables.remove(table); - ClickHouseExpression.ClickHouseJoin.JoinType options; - options = Randomly.fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); - if (options == ClickHouseExpression.ClickHouseJoin.JoinType.NATURAL) { - // NATURAL joins do not have an ON clause - joinClause = null; - } - ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(table, joinClause, - options); + ClickHouseTableReference leftTable = leftTables + .get((int) Randomly.getNotCachedInteger(0, leftTables.size() - 1)); + ClickHouseTableReference rightTable = new ClickHouseTableReference(Randomly.fromList(tables), + "right_" + i); + ClickHouseExpression.ClickHouseJoinOnClause joinClause = generateJoinClause(leftTable, rightTable); + ClickHouseExpression.ClickHouseJoin.JoinType options = Randomly + .fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); + ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(leftTable, rightTable, + options, joinClause); joinStatements.add(j); + leftTables.add(rightTable); } - } return joinStatements; } @@ -111,7 +297,8 @@ protected boolean canGenerateColumnOfType(ClickHouseLancerDataType type) { } @Override - public ClickHouseExpression generateConstant(ClickHouseLancerDataType type) { + public ClickHouseExpression generateConstant(ClickHouseLancerDataType genType) { + ClickHouseLancerDataType type = (genType == null) ? ClickHouseLancerDataType.getRandom() : genType; switch (type.getType()) { case Int8: case UInt8: @@ -121,33 +308,30 @@ public ClickHouseExpression generateConstant(ClickHouseLancerDataType type) { case UInt32: case Int64: case UInt64: - return ClickHouseConstant.createIntConstant(type.getType(), globalState.getRandomly().getInteger()); + return ClickHouseCreateConstant.createIntConstant(type.getType(), globalState.getRandomly().getInteger()); case Float32: - return ClickHouseConstant.createFloat32Constant((float) globalState.getRandomly().getDouble()); + return ClickHouseCreateConstant.createFloat32Constant((float) globalState.getRandomly().getDouble()); case Float64: - return ClickHouseConstant.createFloat64Constant(globalState.getRandomly().getDouble()); + return ClickHouseCreateConstant.createFloat64Constant(globalState.getRandomly().getDouble()); case String: - return ClickHouseConstant.createStringConstant(globalState.getRandomly().getString()); + return ClickHouseCreateConstant.createStringConstant(globalState.getRandomly().getString()); default: throw new AssertionError(); } } public ClickHouseExpression getHavingClause() { - allowAggregateFunctions = true; - return generateExpression(new ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + return generateAggregate(); } public ClickHouseAggregate generateArgsForAggregate(ClickHouseDataType dataType, ClickHouseAggregate.ClickHouseAggregateFunction agg) { - List types = agg.getTypes(dataType); - List args = new ArrayList<>(); - for (ClickHouseDataType argType : types) { - this.allowAggregateFunctions = false; - args.add(generateExpression(new ClickHouseLancerDataType(argType))); - this.allowAggregateFunctions = true; - } - return new ClickHouseAggregate(args, agg); + ClickHouseDataType type = agg.getType(dataType); + this.allowAggregateFunctions = false; + ClickHouseExpression arg = generateExpression(new ClickHouseLancerDataType(type)); + this.allowAggregateFunctions = true; + + return new ClickHouseAggregate(arg, agg); } public ClickHouseExpressionGenerator allowAggregates(boolean value) { @@ -156,13 +340,111 @@ public ClickHouseExpressionGenerator allowAggregates(boolean value) { } public ClickHouseExpression generateAggregate() { - return getAggregate(ClickHouseLancerDataType.getRandom().getType()); + return generateAggregateExpressionWithColumns(columnRefs, 3); + } + + @Override + public ClickHouseExpression generatePredicate() { + return generateExpressionWithColumns(columnRefs, 3); + } + + @Override + public ClickHouseExpression negatePredicate(ClickHouseExpression predicate) { + return new ClickHouseUnaryPrefixOperation(predicate, ClickHouseUnaryPrefixOperator.NOT); + } + + @Override + public ClickHouseExpression isNull(ClickHouseExpression expr) { + return new ClickHouseUnaryPostfixOperation(expr, ClickHouseUnaryPostfixOperator.IS_NULL, false); + } + + @Override + public ClickHouseExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.tables = tables.getTables(); + this.columns = tables.getColumns(); + return this; } - private ClickHouseExpression getAggregate(ClickHouseDataType dataType) { - List aggregates = ClickHouseAggregate.ClickHouseAggregateFunction - .getAggregates(dataType); - ClickHouseAggregate.ClickHouseAggregateFunction agg = Randomly.fromList(aggregates); - return generateArgsForAggregate(dataType, agg); + @Override + public ClickHouseExpression generateBooleanExpression() { + List columnRefs = columns.stream() + .map(c -> c.asColumnReference(c.getTable().getName())).collect(Collectors.toList()); + return generateExpressionWithColumns(columnRefs, 5); + } + + @Override + public ClickHouseSelect generateSelect() { + return new ClickHouseSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + if (globalState.getClickHouseOptions().testJoins && Randomly.getBoolean()) { + return joinStatements; + } + List leftTables = new ArrayList<>(); + leftTables.add(new ClickHouseTableReference(tables.get(0), null)); + if (Randomly.getBoolean() && !tables.isEmpty()) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + for (int i = 0; i < nrJoinClauses; i++) { + ClickHouseTableReference leftTable = leftTables + .get((int) Randomly.getNotCachedInteger(0, leftTables.size() - 1)); + ClickHouseTableReference rightTable = new ClickHouseTableReference(Randomly.fromList(tables), + "right_" + i); + ClickHouseExpression.ClickHouseJoinOnClause joinClause = generateJoinClause(leftTable, rightTable); + ClickHouseExpression.ClickHouseJoin.JoinType options = Randomly + .fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); + ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(leftTable, rightTable, + options, joinClause); + joinStatements.add(j); + leftTables.add(rightTable); + } + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new ClickHouseTableReference(t, null)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(ClickHouseSelect select, ClickHouseExpression whereCondition, + boolean shouldUseAggregate) { + List filteredColumns = Randomly.extractNrRandomColumns(columns, + (int) Randomly.getNotCachedInteger(1, columns.size())); + if (shouldUseAggregate) { + ClickHouseAggregate aggr = new ClickHouseAggregate( + new ClickHouseColumnReference(ClickHouseColumn.createDummy("*", null), null, null), + ClickHouseAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + select.setFetchColumns(filteredColumns.stream().map(c -> c.asColumnReference(c.getTable().getName())) + .collect(Collectors.toList())); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(ClickHouseSelect select, ClickHouseExpression whereCondition) { + ClickHouseExpression inner = new ClickHouseAliasOperation(whereCondition, "check"); + + select.setFetchColumns(List.of(inner)); + select.setWhereClause(null); + return "SELECT SUM(check <> 0) FROM (" + select.asString() + ") as res"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new ClickHouseColumnReference(ClickHouseColumn.createDummy("*", null), null, null)); + } + List columnReferences = columns.stream() + .map(c -> c.asColumnReference(c.getTable().getName())).collect(Collectors.toList()); + return IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> generateExpressionWithColumns(columnReferences, 5)).collect(Collectors.toList()); } } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 3cd4a9435..3951f4b55 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -1,50 +1,46 @@ package sqlancer.clickhouse.gen; -import java.sql.SQLException; -import java.util.HashSet; +import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ClickHouseToStringVisitor; -import sqlancer.gen.AbstractInsertGenerator; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { private final ClickHouseGlobalState globalState; - private final Set errors = new HashSet<>(); private final ClickHouseExpressionGenerator gen; public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { this.globalState = globalState; gen = new ClickHouseExpressionGenerator(globalState); - errors.add("Cannot insert NULL value into a column of type 'Int32'"); // TODO - errors.add("Cannot insert NULL value into a column of type 'String'"); - + errors.add("Cannot insert NULL value into a column of type"); + errors.add("Memory limit"); errors.add("Cannot parse string"); errors.add("Cannot parse Int32 from String, because value is too short"); + errors.add("does not return a value of type UInt8"); + ClickHouseErrors.addExpectedExpressionErrors(errors); } - public static Query getQuery(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseInsertGenerator(globalState).get(); + public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) { + return new ClickHouseInsertGenerator(globalState).getStatement(); } - private Query get() { + @Override + public void buildStatement() { ClickHouseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyColumnSubset(); - sb.append("INSERT INTO "); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); - return new QueryAdapter(sb.toString(), errors); + List columns = Collections.emptyList(); + while (columns.isEmpty()) { + columns = table.getRandomNonEmptyColumnSubset().stream().filter(c -> !c.isAlias() && !c.isMaterialized()) + .collect(Collectors.toList()); + } + buildInsertInto(table.getName(), columns); } @Override diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index a2f355174..3df5c5524 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -1,61 +1,119 @@ package sqlancer.clickhouse.gen; -import java.util.HashSet; -import java.util.Set; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; +import sqlancer.clickhouse.ClickHouseErrors; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseToStringVisitor; +import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseTableGenerator { private enum ClickHouseEngine { - TinyLog, StripeLog, Log, Memory, MergeTree + // TinyLog, StripeLog, + Log, Memory, MergeTree } - StringBuilder sb = new StringBuilder("CREATE TABLE "); - Set errors = new HashSet<>(); + private final StringBuilder sb = new StringBuilder(); + private final String tableName; + private int columnId; + private final List columnNames = new ArrayList<>(); + private final List columns = new ArrayList<>(); + private final ClickHouseProvider.ClickHouseGlobalState globalState; - public Query getQuery(ClickHouseGlobalState globalState) { + public ClickHouseTableGenerator(String tableName, ClickHouseProvider.ClickHouseGlobalState globalState) { + this.tableName = tableName; + this.globalState = globalState; + } + + public static SQLQueryAdapter createTableStatement(String tableName, + ClickHouseProvider.ClickHouseGlobalState globalState) { + ClickHouseTableGenerator chTableGenerator = new ClickHouseTableGenerator(tableName, globalState); + chTableGenerator.start(); + ExpectedErrors errors = new ExpectedErrors(); + ClickHouseErrors.addExpectedExpressionErrors(errors); + return new SQLQueryAdapter(chTableGenerator.sb.toString(), errors, true); + } + + public void start() { ClickHouseEngine engine = Randomly.fromOptions(ClickHouseEngine.values()); - sb.append(globalState.getSchema().getFreeTableName()); - sb.append("("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState).allowAggregates(false); + sb.append("CREATE "); + sb.append("TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(this.globalState.getDatabaseName()); + sb.append("."); + sb.append(this.tableName); + sb.append(" ("); + int nrColumns = 1 + Randomly.smallNumber(); + for (int i = 0; i < nrColumns; i++) { + columns.add(ClickHouseSchema.ClickHouseColumn.createDummy(ClickHouseCommon.createColumnName(i), null)); + } + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } - sb.append("c"); - sb.append(i); - sb.append(" "); - if (Randomly.getBoolean()) { - sb.append(ClickHouseLancerDataType.getRandom()); - } else { - // sb.append("Nullable("); - sb.append(ClickHouseLancerDataType.getRandom()); - // sb.append(")"); + String columnName = ClickHouseCommon.createColumnName(columnId); + ClickHouseColumnBuilder columnBuilder = new ClickHouseColumnBuilder(); + sb.append(columnBuilder.createColumn(columnName, globalState, columns)); + columnNames.add(columnName); + columnId++; + } + if (Randomly.getBooleanWithSmallProbability()) { + for (int i = 0; i < Randomly.smallNumber(); i++) { + addColumnsConstraint(gen); } - potentiallyAppendCodec(); } sb.append(") ENGINE = "); sb.append(engine); sb.append("("); - sb.append(")"); + sb.append(") "); if (engine == ClickHouseEngine.MergeTree) { - sb.append(" ORDER BY tuple()"); + if (Randomly.getBoolean()) { + sb.append(" ORDER BY "); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); + sb.append(ClickHouseToStringVisitor.asString(expr)); + } else { + sb.append(" ORDER BY tuple() "); + } + + if (Randomly.getBoolean()) { + sb.append(" PARTITION BY "); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); + sb.append(ClickHouseToStringVisitor.asString(expr)); + } + if (Randomly.getBoolean()) { + sb.append(" SAMPLE BY "); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); + sb.append(ClickHouseToStringVisitor.asString(expr)); + } + // Suppress index sanity checks https://github.com/sqlancer/sqlancer/issues/788 + sb.append(" SETTINGS allow_suspicious_indices=1"); + // TODO: PRIMARY KEY } - sb.append(";"); - return new QueryAdapter(sb.toString(), errors); + } - private void potentiallyAppendCodec() { - if (Randomly.getBoolean()) { - sb.append(" CODEC("); - errors.add(" in memory is not of fixed size"); - sb.append(Randomly.fromOptions("NONE", "ZSTD", "LZ4HC")); - sb.append(")"); + private void addColumnsConstraint(ClickHouseExpressionGenerator gen) { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + sb.append(","); + sb.append(" CONSTRAINT "); + sb.append(ClickHouseCommon.createConstraintName(i)); + sb.append(" CHECK "); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 2); + sb.append(ClickHouseToStringVisitor.asString(expr)); } } - } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java new file mode 100644 index 000000000..cedac9c51 --- /dev/null +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -0,0 +1,87 @@ +package sqlancer.clickhouse.oracle.tlp; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.clickhouse.ClickHouseErrors; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; + +public class ClickHouseTLPAggregateOracle extends ClickHouseTLPBase { + + public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState state) { + super(state); + ClickHouseErrors.addExpectedExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); + } + + ClickHouseAggregate.ClickHouseAggregateFunction windowFunction = Randomly.fromOptions( + ClickHouseAggregate.ClickHouseAggregateFunction.MIN, + ClickHouseAggregate.ClickHouseAggregateFunction.MAX, + ClickHouseAggregate.ClickHouseAggregateFunction.SUM); + + ClickHouseAggregate aggregate = new ClickHouseAggregate(gen.generateExpressionWithColumns(columns, 6), + windowFunction); + select.setFetchColumns(Arrays.asList(aggregate)); + + String originalQuery = ClickHouseVisitor.asString(select); + originalQuery += " SETTINGS aggregate_functions_null_for_empty = 1"; + + select.setFetchColumns(Arrays.asList(new ClickHouseAliasOperation(aggregate, "aggr"))); + + select.setWhereClause(predicate); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByClause(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); + } + if (Randomly.getBoolean()) { + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); + } + + String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; + metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; + select.setWhereClause(negatedPredicate); + metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; + select.setWhereClause(isNullPredicate); + metamorphicText += ClickHouseVisitor.asString(select); + metamorphicText += ")"; + metamorphicText += " SETTINGS aggregate_functions_null_for_empty = 1"; + List firstResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQuery, errors, state); + + List secondResult = ComparatorHelper.getResultSetFirstColumnAsString(metamorphicText, errors, state); + + state.getState().getLocalState() + .log("--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " + secondResult + + "\n--first size " + firstResult.size() + "\n--second size " + secondResult.size()); + + if (firstResult.size() != secondResult.size()) { + throw new AssertionError(); + } else if (firstResult.isEmpty() || firstResult.equals(secondResult)) { + return; + } else if (firstResult.size() == 1 && secondResult.size() == 1) { + if (firstResult.get(0).equals(secondResult.get(0))) { + return; + } else if (!ComparatorHelper.isEqualDouble(firstResult.get(0), secondResult.get(0))) { + throw new AssertionError(); + } + } else { + throw new AssertionError(); + } + } + +} diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 5c86e3bab..9374442ff 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -1,77 +1,81 @@ package sqlancer.clickhouse.oracle.tlp; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import static java.lang.Math.min; +import static java.util.stream.IntStream.range; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.TestOracle; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; -import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator; -import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; -import sqlancer.clickhouse.gen.ClickHouseCommon; +import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -public class ClickHouseTLPBase implements TestOracle { +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; - final ClickHouseGlobalState state; - final Set errors = new HashSet<>(); +public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { - ClickHouseSchema s; - ClickHouseTables targetTables; + ClickHouseSchema schema; + List columns; ClickHouseExpressionGenerator gen; ClickHouseSelect select; - ClickHouseExpression predicate; - ClickHouseExpression negatedPredicate; - ClickHouseExpression isNullPredicate; public ClickHouseTLPBase(ClickHouseGlobalState state) { - this.state = state; + super(state); ClickHouseErrors.addExpectedExpressionErrors(errors); - ClickHouseErrors.addQueryErrors(errors); } @Override public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new ClickHouseExpressionGenerator(state).setColumns(targetTables.getColumns()); + gen = new ClickHouseExpressionGenerator(state); + schema = state.getSchema(); select = new ClickHouseSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List tableRefs = ClickHouseCommon.getTableRefs(tables, s); - select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); - select.setFromTables(tableRefs); + List tables = schema.getRandomTableNonEmptyTables().getTables(); + ClickHouseTableReference table = new ClickHouseTableReference( + tables.get((int) Randomly.getNotCachedInteger(0, tables.size())), + Randomly.getBoolean() ? "left" : null); + select.setFromClause(table); + columns = table.getColumnReferences(); + + if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { + List joinStatements = gen.getRandomJoinClauses(table, tables); + columns.addAll(joinStatements.stream().flatMap(j -> j.getRightTable().getColumnReferences().stream()) + .collect(Collectors.toList())); + select.setJoinClauses(joinStatements); + } + gen.addColumns(columns); + int small = Randomly.smallNumber(); + List from = range(0, 1 + small) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList()); + select.setFetchColumns(from); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new ClickHouseUnaryPrefixOperation(predicate, ClickHouseUnaryPrefixOperator.NOT); - isNullPredicate = new ClickHouseUnaryPostfixOperation(predicate, ClickHouseUnaryPostfixOperator.IS_NULL, false); + initializeTernaryPredicateVariants(); + // Smoke check + String query = ClickHouseVisitor.asString(select); + ComparatorHelper.getResultSetFirstColumnAsString(query, errors, state); } - List generateFetchColumns() { - List columns = new ArrayList<>(); - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); - return columns; + List generateFetchColumns(List columns) { + List list = Randomly.extractNrRandomColumns(columns, + min(1 + Randomly.smallNumber(), columns.size())); + return list.stream().map(c -> (ClickHouseExpression) c).collect(Collectors.toList()); } - ClickHouseExpression generatePredicate() { - return gen.generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + @Override + protected ExpressionGenerator getGen() { + return gen; } } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java similarity index 65% rename from src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java rename to src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java index ef0d508b6..3165306ae 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPDistinctOracle.java @@ -5,29 +5,25 @@ import java.util.List; import sqlancer.ComparatorHelper; -import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.ClickHouseSelect; -public class ClickHouseTLPWhereOracle extends ClickHouseTLPBase { +public class ClickHouseTLPDistinctOracle extends ClickHouseTLPBase { - public ClickHouseTLPWhereOracle(ClickHouseProvider.ClickHouseGlobalState state) { + public ClickHouseTLPDistinctOracle(ClickHouseProvider.ClickHouseGlobalState state) { super(state); } @Override public void check() throws SQLException { super.check(); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } + select.setSelectType(ClickHouseSelect.SelectType.DISTINCT); + select.setWhereClause(null); String originalQueryString = ClickHouseVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); - } select.setWhereClause(predicate); String firstQueryString = ClickHouseVisitor.asString(select); select.setWhereClause(negatedPredicate); @@ -35,9 +31,10 @@ public void check() throws SQLException { select.setWhereClause(isNullPredicate); String thirdQueryString = ClickHouseVisitor.asString(select); List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, false, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } + } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index 5f4abdc42..e8881cb84 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -1,17 +1,17 @@ package sqlancer.clickhouse.oracle.tlp; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseVisitor; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - public class ClickHouseTLPGroupByOracle extends ClickHouseTLPBase { public ClickHouseTLPGroupByOracle(ClickHouseProvider.ClickHouseGlobalState state) { @@ -21,7 +21,10 @@ public ClickHouseTLPGroupByOracle(ClickHouseProvider.ClickHouseGlobalState state @Override public void check() throws SQLException { super.check(); - select.setGroupByClause(select.getFetchColumns()); + List groupByColumns = IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList()); + + select.setGroupByClause(groupByColumns); select.setWhereClause(null); String originalQueryString = ClickHouseVisitor.asString(select); @@ -34,18 +37,9 @@ public void check() throws SQLException { select.setWhereClause(isNullPredicate); String thirdQueryString = ClickHouseVisitor.asString(select); List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, - secondQueryString, thirdQueryString, combinedString, false, state, errors); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } - - @Override - List generateFetchColumns() { - List columns; - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); - return columns; - } - } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index 40393fd59..a6f3e0cdb 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -1,57 +1,54 @@ package sqlancer.clickhouse.oracle.tlp; +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; +import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseCommon; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; - -import java.sql.SQLException; -import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; public class ClickHouseTLPHavingOracle extends ClickHouseTLPBase { public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) { super(state); - ClickHouseErrors.addGroupingErrors(errors); + ClickHouseErrors.addExpectedExpressionErrors(errors); } @Override public void check() throws SQLException { - ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables targetTables = s.getRandomTableNonEmptyTables(); - List groupByColumns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); - List columns = targetTables.getColumns(); - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); - ClickHouseExpressionGenerator aggrGen = new ClickHouseExpressionGenerator(state).allowAggregates(true) - .setColumns(columns); - ClickHouseSelect select = new ClickHouseSelect(); - select.setFetchColumns(aggrGen.generateExpressions(Randomly.smallNumber() + 1)); - List tables = targetTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List from = ClickHouseCommon.getTableRefs(tables, state.getSchema()); - select.setJoinClauses(joinStatements); + super.check(); + select.setFetchColumns(IntStream.range(0, Randomly.smallNumber() + 1) + .mapToObj(i -> gen.generateAggregateExpressionWithColumns(columns, 3)).collect(Collectors.toList())); select.setSelectType(ClickHouseSelect.SelectType.ALL); - select.setFromTables(from); // TODO order by? + + List groupByColumns = IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 6)).collect(Collectors.toList()); + select.setGroupByClause(groupByColumns); select.setHavingClause(null); String originalQueryString = ClickHouseVisitor.asString(select); + originalQueryString += " SETTINGS aggregate_functions_null_for_empty=1, enable_optimize_predicate_expression=0"; // https://github.com/ClickHouse/ClickHouse/issues/12264 List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - ClickHouseExpression predicate = aggrGen.getHavingClause(); + List aggregateExprs = select.getFetchColumns().stream() + .filter(p -> p instanceof ClickHouseAggregate).collect(Collectors.toList()); + if (aggregateExprs.isEmpty()) { + throw new IgnoreMeException(); + } + ClickHouseExpression predicate = gen.generateExpressionWithExpression(aggregateExprs, 6); select.setHavingClause(predicate); String firstQueryString = ClickHouseVisitor.asString(select); select.setHavingClause(new ClickHouseUnaryPrefixOperation(predicate, @@ -61,6 +58,7 @@ public void check() throws SQLException { ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.IS_NULL, false)); String thirdQueryString = ClickHouseVisitor.asString(select); String combinedString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " + thirdQueryString; + combinedString += " SETTINGS aggregate_functions_null_for_empty=1, enable_optimize_predicate_expression=0"; // https://github.com/ClickHouse/ClickHouse/issues/12264 List secondResultSet = ComparatorHelper.getResultSetFirstColumnAsString(combinedString, errors, state); if (state.getOptions().logEachSelect()) { state.getLogger().writeCurrent(originalQueryString); diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 332f51c4c..86d88ee21 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -3,13 +3,70 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/46915 - public static boolean bug46915 = true; + public static boolean bug46915; // https://github.com/cockroachdb/cockroach/issues/45703 - public static boolean bug45703 = true; + public static boolean bug45703; // https://github.com/cockroachdb/cockroach/issues/44757 - public static boolean bug44757 = true; + public static boolean bug44757; + + // https://github.com/cockroachdb/cockroach/issues/83792 + public static boolean bug83792 = true; + + // https://github.com/cockroachdb/cockroach/issues/83874 + public static boolean bug83874 = true; + + // https://github.com/cockroachdb/cockroach/issues/83973 + public static boolean bug83973; + + // https://github.com/cockroachdb/cockroach/issues/83976 + public static boolean bug83976; + + // The following bug is closed, but leave it enabled until + // the underlying interval issue is resolved. + // https://github.com/cockroachdb/cockroach/issues/84078 + // https://github.com/cockroachdb/cockroach/issues/84154 + public static boolean bug84154 = true; + + // https://github.com/cockroachdb/cockroach/issues/85356 + public static boolean bug85356; + + // https://github.com/cockroachdb/cockroach/issues/85371 + public static boolean bug85371; + + // https://github.com/cockroachdb/cockroach/issues/85389 + public static boolean bug85389; + + // https://github.com/cockroachdb/cockroach/issues/85390 + public static boolean bug85390; + + // https://github.com/cockroachdb/cockroach/issues/85393 + public static boolean bug85393; + + // https://github.com/cockroachdb/cockroach/issues/85394 + public static boolean bug85394 = true; + + // https://github.com/cockroachdb/cockroach/issues/85441 + public static boolean bug85441; + + // https://github.com/cockroachdb/cockroach/issues/85499 + public static boolean bug85499; + + // https://github.com/cockroachdb/cockroach/issues/88037 + public static boolean bug88037; + + // https://github.com/cockroachdb/cockroach/issues/85230 + public static boolean bug85230 = true; + + // https://github.com/cockroachdb/cockroach/issues/131640 + public static boolean bug131640 = true; + + // https://github.com/cockroachdb/cockroach/issues/131647 + public static boolean bug131647 = true; + + // https://github.com/cockroachdb/cockroach/issues/131875 + public static boolean bug131875 = true; private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 87cb561fd..134ad35aa 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -1,13 +1,18 @@ package sqlancer.cockroachdb; -import java.util.Set; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; public final class CockroachDBErrors { private CockroachDBErrors() { } - public static void addExpressionErrors(Set errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add(" non-streaming operator encountered when vectorize=auto"); if (CockroachDBBugs.bug46915) { @@ -19,7 +24,8 @@ public static void addExpressionErrors(Set errors) { } errors.add("exceeds supported timestamp bounds"); - + errors.add("expected STORED COMPUTED COLUMN expression to have type bytes"); + errors.add("volatile functions are not allowed in STORED COMPUTED COLUMN"); errors.add("cannot cast negative integer to bit varying with unbounded width"); errors.add("negative value for LIMIT"); @@ -33,9 +39,9 @@ public static void addExpressionErrors(Set errors) { errors.add("expected -9223372036854775809 to be of type int, found type decimal"); errors.add("to be of type int4, found type decimal"); - errors.add("as type bool: invalid bool value"); - errors.add("as type int: strconv.ParseInt"); - errors.add("as type float: strconv.ParseFloat: parsing"); + errors.add("as type bool"); + errors.add("as type int"); + errors.add("as type float"); errors.add("is not in select list"); errors.add("non-integer constant in ORDER BY"); @@ -68,14 +74,16 @@ public static void addExpressionErrors(Set errors) { errors.add("unsupported comparison operator: = "); errors.add("unsupported comparison operator: <= "); errors.add("to be of type string, found type bytes"); - errors.add("unknown signature: left(string, int) (desired )"); errors.add("unknown signature: bit_length(collatedstring"); - errors.add("ERROR: unknown signature: left(collatedstring"); errors.add("unsupported comparison operator: !~ NOT LIKE != "); errors.add("expected DEFAULT expression to have type bytes"); + errors.add("expected DEFAULT (in CREATE TABLE) expression to have type bytes"); + errors.add("expected DEFAULT (in CREATE VIEW) expression to have type bytes"); + errors.add("expected DEFAULT (in SET DEFAULT) expression to have type bytes"); + errors.add("expected DEFAULT (in ADD COLUMN) expression to have type bytes"); errors.add("value type string doesn't match type bytes of column"); errors.add("as decimal, found type: int"); errors.add("to be of type decimal, found type float"); @@ -92,29 +100,23 @@ public static void addExpressionErrors(Set errors) { errors.add("LOOKUP can only be used with INNER or LEFT joins"); // TODO errors.add("ambiguous binary operator: || "); - errors.add(" ERROR: unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); - errors.add("incompatible value type: unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); + errors.add("unsupported binary operator"); errors.add("parsing as type timestamp: empty or blank input"); errors.add("parsing as type timestamp: field"); errors.add("as type time"); errors.add("as TimeTZ"); errors.add("as type decimal"); - addIntervalTypeErrors(errors); - addFunctionErrors(errors); - addGroupByErrors(errors); - addJoinTypes(errors); + errors.addAll(getIntervalTypeErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getGroupByErrors()); + errors.addAll(getJoinTypes()); errors.add("as int4, found type: decimal"); errors.add("to be of type int2, found type decimal"); errors.add("to be of type int, found type decimal"); // arithmetic overflows - errors.add("unknown signature: left(string, decimal)"); - errors.add("unknown signature: left(bytes, decimal) (desired )"); + errors.add("unknown signature: left"); errors.add("numeric constant out of int64 range"); - errors.add("unknown signature: overlay(string, string, decimal)"); - errors.add("unknown signature: substring(string, int, decimal)"); - errors.add("unsupported binary operator: + (desired )"); + errors.add("unknown signature: overlay"); + errors.add("unknown signature: substring"); errors.add("unsupported comparison operator"); errors.add("unknown signature: chr(decimal) (desired )"); errors.add("unknown signature: to_english(decimal) (desired )"); @@ -122,25 +124,19 @@ public static void addExpressionErrors(Set errors) { errors.add("incompatible value type: expected rowid to be of type decimal, found type int"); errors.add("unknown signature: to_english(decimal)"); errors.add("unknown signature: chr(decimal)"); - errors.add(" unknown signature: left(string, int2) (desired )"); - errors.add("unknown signature: split_part(string, string, decimal) (desired )"); - errors.add(" unknown signature: substring(string, "); + errors.add("unknown signature: split_part"); errors.add("division by zero"); errors.add("as int, found type: decimal"); errors.add("value type decimal doesn't match type int2 "); errors.add("has type decimal"); errors.add("to be of type decimal, found type int"); errors.add("value type decimal doesn't match type int"); - errors.add("unknown signature: substring(string, decimal, int)"); - errors.add("unsupported binary operator: / (desired )"); errors.add("(desired )"); errors.add("(desired )"); errors.add("(desired )"); errors.add("found type: decimal"); errors.add("(desired )"); errors.add("unknown signature: to_hex(decimal)"); - errors.add("unknown signature: split_part(string, string, decimal)"); - errors.add("unknown signature: left(bytes, decimal)"); errors.add("division undefined"); errors.add("decimal out of range"); errors.add("unknown signature: xor_agg(decimal)"); @@ -150,19 +146,50 @@ public static void addExpressionErrors(Set errors) { errors.add("exists but is not a directory"); // TODO - errors.add("could not parse JSON: trailing characters after JSON document"); - errors.add("could not parse JSON: unable to decode JSON: invalid character"); - errors.add("could not parse JSON: unable to decode JSON: EOF"); - errors.add("could not parse JSON: unable to decode JSON: unexpected EOF"); + errors.add("could not parse JSON"); errors.add("can't order by column type jsonb"); + errors.add("odd length hex string"); // TODO: better control what is generated in a view errors.add("aggregate functions are not allowed in GROUP BY"); errors.add(" must appear in the GROUP BY clause or be used in an aggregate function"); - if (CockroachDBBugs.bug44757) { + if (CockroachDBBugs.bug83874) { errors.add("no builtin aggregate"); } + if (CockroachDBBugs.bug83792) { + errors.add("comparison overload not found"); + } + if (CockroachDBBugs.bug83973) { + errors.add("invalid memory address"); + } + if (CockroachDBBugs.bug83976) { + errors.add("cannot execute distinct on no columns"); + } + if (CockroachDBBugs.bug85356) { + errors.add("inconsistent Case return types"); + } + if (CockroachDBBugs.bug85371) { + errors.add("index out of range"); + } + if (CockroachDBBugs.bug85389) { + errors.add("no volatility for cast decimal"); + } + if (CockroachDBBugs.bug85390) { + errors.add("lookup for ComparisonExpr"); + } + if (CockroachDBBugs.bug85393) { + errors.add("no output column equivalent to"); + } + if (CockroachDBBugs.bug85441) { + errors.add("cannot cast jsonb numeric to type bool"); + } + if (CockroachDBBugs.bug85499) { + errors.add("estimated row count must be non-zero"); + } + if (CockroachDBBugs.bug88037) { + errors.add("expected required columns to be a subset of output columns"); + } errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; @@ -179,10 +206,21 @@ public static void addExpressionErrors(Set errors) { errors.add("argument of OFFSET must be type int, not type decimal"); errors.add("ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list"); - addArrayErrors(errors); + errors.add("incompatible IF expressions"); + + errors.addAll(getArrayErrors()); + errors.addAll(getComputedColumnErrors()); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); } - private static void addArrayErrors(Set errors) { + private static List getArrayErrors() { + ArrayList errors = new ArrayList<>(); + // arrays errors.add("cannot determine type of empty array"); errors.add("unknown signature: max(unknown[])"); @@ -223,25 +261,34 @@ private static void addArrayErrors(Set errors) { errors.add("unimplemented: nested arrays not supported"); // e.g., casting a string {{1}} to an array errors.add("malformed array"); - errors.add("https://github.com/cockroachdb/cockroach/issues/35707"); // arrays don't support ORDER BY - errors.add("as bytes[], found type: varbit[]"); errors.add("to be of type decimal[], found type float[]"); errors.add("to be of type int[], found type decimal[]"); errors.add("to be of type unknown[]"); // IF with null array + + return errors; } - private static void addIntervalTypeErrors(Set errors) { + private static List getIntervalTypeErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("overflow during Encode"); - errors.add("as type interval"); + errors.add("type interval"); + + return errors; } - private static void addJoinTypes(Set errors) { + private static List getJoinTypes() { + ArrayList errors = new ArrayList<>(); + errors.add("JOIN/USING types"); + + return errors; } - private static void addGroupByErrors(Set errors) { + private static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); errors.add("non-integer constant in GROUP BY"); // https://github.com/cockroachdb/cockroach/pull/46649 -> aggregates on NULL are @@ -263,13 +310,14 @@ private static void addGroupByErrors(Set errors) { errors.add("unknown signature: abs(string)"); errors.add("unknown signature: acos(string)"); + return errors; } - private static void addFunctionErrors(Set errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); // functions errors.add("abs of min integer value (-9223372036854775808) not defined"); // ABS errors.add("the input string must not be empty"); // ASCII - errors.add("unknown signature: substring(string, decimal)"); // overflow errors.add("overlay(): non-positive substring length not allowed"); // overlay errors.add("non-positive substring length not allowed"); // overlay errors.add("lpad(): requested length too large"); // lpad @@ -282,10 +330,30 @@ private static void addFunctionErrors(Set errors) { errors.add("substring(): negative substring length"); // substring errors.add("negative substring length"); // substring errors.add("must be greater than zero"); // split_part + + return errors; } - public static void addTransactionErrors(Set errors) { + public static List getTransactionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("current transaction is aborted"); + + return errors; + } + + public static void addTransactionErrors(ExpectedErrors errors) { + errors.addAll(getTransactionErrors()); + } + + private static List getComputedColumnErrors() { + ArrayList errors = new ArrayList<>(); + + // computed columns + errors.add("computed column expressions cannot reference computed columns"); + errors.add("STORED COMPUTED COLUMN expression cannot reference computed columns"); + + return errors; } } diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index f822f0086..ce8a207d6 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -1,100 +1,38 @@ package sqlancer.cockroachdb; -import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.MainOptions.DBMSConverter; -import sqlancer.TestOracle; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPJoinOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; +import sqlancer.DBMSSpecificOptions; -@Parameters(separators = "=", commandDescription = "Test CockroachDB") -public class CockroachDBOptions { +@Parameters(separators = "=", commandDescription = "CockroachDB (default port: " + CockroachDBOptions.DEFAULT_PORT + + " default host: " + CockroachDBOptions.DEFAULT_HOST + ")") +public class CockroachDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 26257; - @Parameter(names = "--oracle", converter = DBMSConverter.class) - public CockroachDBOracle oracle = CockroachDBOracle.NOREC; - - public enum CockroachDBOracle { - NOREC { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBNoRECOracle(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPAggregateOracle(globalState); - } - - }, - GROUP_BY { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPGroupByOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPHavingOracle(globalState); - } - }, - WHERE { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPWhereOracle(globalState); - } - }, - DISTINCT { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPDistinctOracle(globalState); - } - }, - EXTENDED_WHERE { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPExtendedWhereOracle(globalState); - } - }, - JOIN { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPJoinOracle(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPOracle(globalState); - } - }; - - public abstract TestOracle create(CockroachDBGlobalState globalState) throws SQLException; - - } + @Parameter(names = "--oracle") + public CockroachDBOracleFactory oracle = CockroachDBOracleFactory.NOREC; @Parameter(names = { "--test-hash-indexes" }, description = "Test the USING HASH WITH BUCKET_COUNT=n_buckets option in CREATE INDEX") public boolean testHashIndexes = true; @Parameter(names = { "--test-temp-tables" }, description = "Test TEMPORARY tables") - public boolean testTempTables = true; + public boolean testTempTables; // default: false https://github.com/cockroachdb/cockroach/issues/85388 - @Parameter(names = { - "--increased-vectorization" }, description = "Generate VECTORIZE=on with a higher probability (which found a number of bugs in the past)") - public boolean makeVectorizationMoreLikely = true; + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + + @Override + public List getTestOracleFactory() { + return Arrays.asList(oracle); + } } diff --git a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java new file mode 100644 index 000000000..96fbc22ce --- /dev/null +++ b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java @@ -0,0 +1,138 @@ +package sqlancer.cockroachdb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import sqlancer.IgnoreMeException; +import sqlancer.OracleFactory; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; +import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; + +public enum CockroachDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with(CockroachDBErrors.getTransactionErrors()).with("unable to vectorize execution plan") // SET + // vectorize=experimental_always; + .with(" mismatched physical types at index") // SET vectorize=experimental_always; + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + AGGREGATE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPAggregateOracle(globalState); + } + + }, + GROUP_BY { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPGroupByOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPHavingOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with("GROUP BY term out of range").build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + DISTINCT { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPDistinctOracle(globalState); + } + }, + EXTENDED_WHERE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPExtendedWhereOracle(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(WHERE.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(EXTENDED_WHERE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(1); + if (content.contains("count:")) { + try { + long number = Long.parseLong(content.split("count: ")[1].split(" ")[0].replace(",", "")); + return Optional.of(number); + } catch (Exception e) { // To avoid the situation that no number is found + } + } + return Optional.empty(); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String content = rs.getString(1); + if (content.contains("• ")) { + String operation = content.split("• ")[1].split(" ")[0]; + if (CockroachDBBugs.bug131875 && (operation.equals("distinct") || operation.equals("limit"))) { + throw new IgnoreMeException(); + } + return Optional.of(operation); + } + return Optional.empty(); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }; + +} diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 6e7f7352d..66e80fd59 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -6,29 +6,26 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.GlobalState; +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; -import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.StateToReproduce; -import sqlancer.TestOracle; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; import sqlancer.cockroachdb.gen.CockroachDBCreateStatisticsGenerator; import sqlancer.cockroachdb.gen.CockroachDBDeleteGenerator; +import sqlancer.cockroachdb.gen.CockroachDBDropTableGenerator; +import sqlancer.cockroachdb.gen.CockroachDBDropViewGenerator; import sqlancer.cockroachdb.gen.CockroachDBIndexGenerator; import sqlancer.cockroachdb.gen.CockroachDBInsertGenerator; import sqlancer.cockroachdb.gen.CockroachDBRandomQuerySynthesizer; @@ -39,38 +36,42 @@ import sqlancer.cockroachdb.gen.CockroachDBTruncateGenerator; import sqlancer.cockroachdb.gen.CockroachDBUpdateGenerator; import sqlancer.cockroachdb.gen.CockroachDBViewGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; -public class CockroachDBProvider extends ProviderAdapter { +@AutoService(DatabaseProvider.class) +public class CockroachDBProvider extends SQLProviderAdapter { public CockroachDBProvider() { super(CockroachDBGlobalState.class, CockroachDBOptions.class); } public enum Action { - INSERT(CockroachDBInsertGenerator::insert), // - TRUNCATE(CockroachDBTruncateGenerator::truncate), // + CREATE_TABLE(CockroachDBTableGenerator::generate), CREATE_INDEX(CockroachDBIndexGenerator::create), // + CREATE_VIEW(CockroachDBViewGenerator::generate), // CREATE_STATISTICS(CockroachDBCreateStatisticsGenerator::create), // - SET_SESSION(CockroachDBSetSessionGenerator::create), // - CREATE_INDEX(CockroachDBIndexGenerator::create), // + INSERT(CockroachDBInsertGenerator::insert), // UPDATE(CockroachDBUpdateGenerator::gen), // - CREATE_VIEW(CockroachDBViewGenerator::generate), // + SET_SESSION(CockroachDBSetSessionGenerator::create), // SET_CLUSTER_SETTING(CockroachDBSetClusterSettingGenerator::create), // DELETE(CockroachDBDeleteGenerator::delete), // + TRUNCATE(CockroachDBTruncateGenerator::truncate), // + DROP_TABLE(CockroachDBDropTableGenerator::drop), // + DROP_VIEW(CockroachDBDropViewGenerator::drop), // COMMENT_ON(CockroachDBCommentOnGenerator::comment), // SHOW(CockroachDBShowGenerator::show), // TRANSACTION((g) -> { String s = Randomly.fromOptions("BEGIN", "ROLLBACK", "COMMIT"); - return new QueryAdapter(s, - Arrays.asList("there is no transaction in progress", "there is already a transaction in progress", - "current transaction is aborted", "does not exist" /* interleaved indexes */)); - }), // - EXPLAIN((g) -> { + return new SQLQueryAdapter(s, ExpectedErrors.from("there is no transaction in progress", + "there is already a transaction in progress", "current transaction is aborted")); + }), EXPLAIN((g) -> { StringBuilder sb = new StringBuilder("EXPLAIN "); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); if (Randomly.getBoolean()) { sb.append("("); - sb.append(Randomly.nonEmptySubset("VERBOSE", "TYPES", "OPT", "DISTSQL", "VEC").stream() - .collect(Collectors.joining(", "))); + sb.append(Randomly.fromOptions("VERBOSE", "TYPES", "OPT", "DISTSQL", "VEC")); sb.append(") "); errors.add("cannot set EXPLAIN mode more than once"); errors.add("unable to vectorize execution plan"); @@ -79,12 +80,12 @@ public enum Action { } sb.append(CockroachDBRandomQuerySynthesizer.generate(g, Randomly.smallNumber() + 1)); CockroachDBErrors.addExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); }), // - SCRUB((g) -> new QueryAdapter( + SCRUB((g) -> new SQLQueryAdapter( "EXPERIMENTAL SCRUB table " + g.getSchema().getRandomTable(t -> !t.isView()).getName(), // https://github.com/cockroachdb/cockroach/issues/46401 - Arrays.asList("scrub-fk: column \"t.rowid\" does not exist", + ExpectedErrors.from("scrub-fk: column \"t.rowid\" does not exist", "check-constraint: cannot access temporary tables of other sessions" /* * https:// github. com/ * cockroachdb / cockroach @@ -101,88 +102,66 @@ public enum Action { } else { sb.append(" SPLIT AT VALUES (NULL);"); } - return new QueryAdapter(sb.toString(), Arrays.asList("must be of type")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("must be of type")); }); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } - public Query getQuery(CockroachDBGlobalState state) throws SQLException { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(CockroachDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } - public static class CockroachDBGlobalState extends GlobalState { - - private CockroachDBSchema schema; - - public void setSchema(CockroachDBSchema schema) { - this.schema = schema; - } + public static class CockroachDBGlobalState extends SQLGlobalState { - public CockroachDBSchema getSchema() { - return schema; + @Override + protected CockroachDBSchema readSchema() throws SQLException { + return CockroachDBSchema.fromConnection(getConnection(), getDatabaseName()); } } @Override - public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws SQLException { - Randomly r = new Randomly(); - Connection con = globalState.getConnection(); - String databaseName = globalState.getDatabaseName(); - QueryManager manager = globalState.getManager(); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); + public void generateDatabase(CockroachDBGlobalState globalState) throws Exception { + QueryManager manager = globalState.getManager(); MainOptions options = globalState.getOptions(); - globalState.setSchema(CockroachDBSchema.fromConnection(con, databaseName)); - List standardSettings = new ArrayList<>(); - standardSettings.add("--Don't send automatic bug reports\n" - + "SET CLUSTER SETTING debug.panic_on_failed_assertions = true;"); + standardSettings.add("--Don't send automatic bug reports"); + standardSettings.add("SET CLUSTER SETTING debug.panic_on_failed_assertions = true;"); standardSettings.add("SET CLUSTER SETTING diagnostics.reporting.enabled = false;"); standardSettings.add("SET CLUSTER SETTING diagnostics.reporting.send_crash_reports = false;"); - standardSettings.add("-- Disable the collection of metrics and hope that it helps performance\n" - + "SET CLUSTER SETTING sql.metrics.statement_details.enabled = 'off'"); + standardSettings.add("-- Disable the collection of metrics and hope that it helps performance"); + standardSettings.add("SET CLUSTER SETTING sql.metrics.statement_details.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING sql.metrics.statement_details.plan_collection.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING sql.stats.automatic_collection.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING timeseries.storage.enabled = 'off'"); - if (globalState.getDmbsSpecificOptions().testHashIndexes) { + if (globalState.getDbmsSpecificOptions().testHashIndexes) { standardSettings.add("set experimental_enable_hash_sharded_indexes='on';"); } - if (globalState.getDmbsSpecificOptions().testTempTables) { + if (globalState.getDbmsSpecificOptions().testTempTables) { standardSettings.add("SET experimental_enable_temp_tables = 'on'"); } for (String s : standardSettings) { - manager.execute(new QueryAdapter(s)); + manager.execute(new SQLQueryAdapter(s)); } for (int i = 0; i < Randomly.fromOptions(2, 3); i++) { boolean success = false; do { try { - Query q = CockroachDBTableGenerator.generate(globalState); - success = manager.execute(q); - logger.writeCurrent(state); - try { - logger.getCurrentFileWriter().close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - logger.currentFileWriter = null; + SQLQueryAdapter q = CockroachDBTableGenerator.generate(globalState); + success = globalState.executeStatement(q); } catch (IgnoreMeException e) { // continue trying } } while (!success); - globalState.setSchema(CockroachDBSchema.fromConnection(con, databaseName)); } - logger.writeCurrent(state); int[] nrRemaining = new int[Action.values().length]; List actions = new ArrayList<>(); @@ -192,30 +171,30 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S int nrPerformed = 0; switch (action) { case INSERT: - nrPerformed = r.getInteger(0, options.getMaxNumberInserts()); + nrPerformed = globalState.getRandomly().getInteger(0, options.getMaxNumberInserts()); break; case UPDATE: case SPLIT: - nrPerformed = r.getInteger(0, 3); + nrPerformed = globalState.getRandomly().getInteger(0, 3); break; case EXPLAIN: - nrPerformed = r.getInteger(0, 10); + nrPerformed = globalState.getRandomly().getInteger(0, 10); break; case SHOW: case TRUNCATE: case DELETE: case CREATE_STATISTICS: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case CREATE_VIEW: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case SET_SESSION: case SET_CLUSTER_SETTING: - nrPerformed = r.getInteger(0, 3); + nrPerformed = globalState.getRandomly().getInteger(0, 3); break; case CREATE_INDEX: - nrPerformed = r.getInteger(0, 10); + nrPerformed = globalState.getRandomly().getInteger(0, 10); break; case COMMENT_ON: case SCRUB: @@ -225,6 +204,9 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S */ break; case TRANSACTION: + case CREATE_TABLE: + case DROP_TABLE: + case DROP_VIEW: nrPerformed = 0; // r.getInteger(0, 0); break; default: @@ -239,7 +221,7 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S while (total != 0) { Action nextAction = null; - int selection = r.getInteger(0, total); + int selection = globalState.getRandomly().getInteger(0, total); int previousRange = 0; for (int i = 0; i < nrRemaining.length; i++) { if (previousRange <= selection && selection < previousRange + nrRemaining[i]) { @@ -252,64 +234,53 @@ public void generateAndTestDatabase(CockroachDBGlobalState globalState) throws S assert nextAction != null; assert nrRemaining[nextAction.ordinal()] > 0; nrRemaining[nextAction.ordinal()]--; - Query query = null; + SQLQueryAdapter query = null; try { boolean success; int nrTries = 0; do { query = nextAction.getQuery(globalState); - if (options.logEachSelect()) { - logger.writeCurrent(query.getQueryString()); - } - success = manager.execute(query); + success = globalState.executeStatement(query); } while (!success && nrTries++ < 1000); } catch (IgnoreMeException e) { } - if (query != null && query.couldAffectSchema()) { - globalState.setSchema(CockroachDBSchema.fromConnection(con, databaseName)); - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } + if (query != null && query.couldAffectSchema() && globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); } total--; } - manager.incrementCreateDatabase(); - if (globalState.getDmbsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { - manager.execute(new QueryAdapter("SET vectorize=on;")); - } - TestOracle oracle = globalState.getDmbsSpecificOptions().oracle.create(globalState); - for (int i = 0; i < options.getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == CockroachDBOracleFactory.CERT)) { + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + CockroachDBErrors.addExpressionErrors(errors); + for (CockroachDBTable table : globalState.getSchema().getDatabaseTables()) { + globalState.executeStatement(new SQLQueryAdapter("ANALYZE " + table.getName() + ";", errors)); } } - try { - if (options.logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } @Override - public Connection createDatabase(CockroachDBGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws SQLException { + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = CockroachDBOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = CockroachDBOptions.DEFAULT_PORT; + } String databaseName = globalState.getDatabaseName(); - String url = "jdbc:postgresql://localhost:26257/test"; + String url = String.format("jdbc:postgresql://%s:%d/test", host, port); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - globalState.getState().statements.add(new QueryAdapter("USE test")); - globalState.getState().statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName + " CASCADE")); + globalState.getState().logStatement("USE test"); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); String createDatabaseCommand = "CREATE DATABASE " + databaseName; - globalState.getState().statements.add(new QueryAdapter(createDatabaseCommand)); - globalState.getState().statements.add(new QueryAdapter("USE " + databaseName)); + globalState.getState().logStatement(createDatabaseCommand); + globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } @@ -317,9 +288,9 @@ public Connection createDatabase(CockroachDBGlobalState globalState) throws SQLE s.execute(createDatabaseCommand); } con.close(); - con = DriverManager.getConnection("jdbc:postgresql://localhost:26257/" + databaseName, + con = DriverManager.getConnection(String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - return con; + return new SQLConnection(con); } @Override @@ -327,4 +298,68 @@ public String getDBMSName() { return "cockroachdb"; } + @Override + public String getQueryPlan(String selectStr, CockroachDBGlobalState globalState) throws Exception { + String queryPlan = ""; + String explainQuery = "EXPLAIN (OPT) " + selectStr; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery); + boolean afterProjection = false; // Remove the concrete expression after each Projection operator + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String targetQueryPlan = rs.getString(1).replace("└──", "").replace("├──", "").replace("│", "") + .trim() + ";"; // Unify format + if (afterProjection) { + afterProjection = false; + continue; + } + if (targetQueryPlan.startsWith("projections")) { + afterProjection = true; + } + // Remove all concrete expressions by keywords + if (targetQueryPlan.contains(">") || targetQueryPlan.contains("<") || targetQueryPlan.contains("=") + || targetQueryPlan.contains("*") || targetQueryPlan.contains("+") + || targetQueryPlan.contains("'")) { + continue; + } + queryPlan += targetQueryPlan; + } + } + } catch (AssertionError e) { + throw new AssertionError("Explain failed: " + explainQuery); + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, CockroachDBGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + + @Override + public boolean addRowsToAllTables(CockroachDBGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (CockroachDBTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = CockroachDBInsertGenerator.insert(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 0379f7cb7..cbde577cb 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -1,6 +1,5 @@ package sqlancer.cockroachdb; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -8,14 +7,16 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; -public class CockroachDBSchema extends AbstractSchema { +public class CockroachDBSchema extends AbstractSchema { public enum CockroachDBDataType { @@ -181,6 +182,7 @@ public CockroachDBColumn(String name, CockroachDBCompositeDataType columnType, b this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -207,6 +209,10 @@ public CockroachDBTables getRandomTableNonEmptyTables() { return new CockroachDBTables(Randomly.nonEmptySubset(getDatabaseTables())); } + public CockroachDBTables getRandomTableNonEmptyTables(int nr) { + return new CockroachDBTables(Randomly.nonEmptySubsetLeast(getDatabaseTables(), nr)); + } + private static CockroachDBCompositeDataType getColumnType(String typeString) { if (typeString.endsWith("[]")) { String substring = typeString.substring(0, typeString.length() - 2); @@ -263,7 +269,8 @@ private static CockroachDBCompositeDataType getColumnType(String typeString) { } } - public static class CockroachDBTable extends AbstractTable { + public static class CockroachDBTable + extends AbstractRelationalTable { public CockroachDBTable(String tableName, List columns, List indexes, boolean isView) { @@ -272,24 +279,36 @@ public CockroachDBTable(String tableName, List columns, List< } - public static CockroachDBSchema fromConnection(Connection con, String databaseName) throws SQLException { + public int getIndexCount() { + int count = 0; + for (CockroachDBTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + + public static CockroachDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); CockroachDBTable t = new CockroachDBTable(tableName, databaseColumns, indexes, isView); for (CockroachDBColumn c : databaseColumns) { c.setTable(t); } + // To avoid some situations that columns can not be retrieved. + if (databaseColumns.isEmpty()) { + continue; + } databaseTables.add(t); } return new CockroachDBSchema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { ResultSet tableRs = s.executeQuery( @@ -302,7 +321,7 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getIndexes(Connection con, String tableName) throws SQLException { + private static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format("SHOW INDEX FROM %s", tableName))) { @@ -315,7 +334,7 @@ private static List getIndexes(Connection con, String tableName) thr return indexes; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SHOW COLUMNS FROM " + tableName)) { @@ -332,6 +351,11 @@ private static List getTableColumns(Connection con, String ta isNullable); columns.add(c); } + } catch (SQLException e) { + if (CockroachDBBugs.bug85394 && e.getMessage().contains("incompatible type annotation for ARRAY")) { + return columns; + } + throw e; } } return columns; diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 80e923d69..67abfdbbe 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -15,7 +15,7 @@ import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.visitor.ToStringVisitor; +import sqlancer.common.visitor.ToStringVisitor; public class CockroachDBToStringVisitor extends ToStringVisitor implements CockroachDBVisitor { @@ -102,9 +102,9 @@ public void visit(CockroachDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); @@ -142,54 +142,65 @@ public void visit(CockroachDBJoin join) { switch (join.getJoinType()) { case INNER: sb.append(" INNER "); - potentiallyAddHint(); + potentiallyAddHint(false); sb.append("JOIN "); visit(join.getRightTable()); sb.append(" ON "); visit(join.getOnCondition()); break; - case NATURAL: - sb.append(" NATURAL "); - // potentiallyAddHint(); + case LEFT: + sb.append(" LEFT"); + sb.append(" OUTER "); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); + sb.append(" ON "); + visit(join.getOnCondition()); break; - case CROSS: - sb.append(" CROSS "); - potentiallyAddHint(); + case RIGHT: + sb.append(" RIGHT"); + sb.append(" OUTER "); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); + sb.append(" ON "); + visit(join.getOnCondition()); break; - case OUTER: - sb.append(" "); - switch (join.getOuterType()) { - case FULL: - sb.append("FULL"); - break; - case LEFT: - sb.append("LEFT"); - break; - case RIGHT: - sb.append("RIGHT"); - break; - default: - throw new AssertionError(); - } + case FULL: + sb.append(" FULL"); sb.append(" OUTER "); - potentiallyAddHint(); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); sb.append(" ON "); visit(join.getOnCondition()); break; + case CROSS: + sb.append(" CROSS "); + potentiallyAddHint(false); + sb.append("JOIN "); + visit(join.getRightTable()); + break; + case NATURAL: + sb.append(" NATURAL "); + // potentiallyAddHint(false); + sb.append("JOIN "); + visit(join.getRightTable()); + break; default: throw new AssertionError(); } } - private void potentiallyAddHint() { + private void potentiallyAddHint(boolean isOuter) { if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions("HASH", "MERGE", "LOOKUP")); + return; + } else { + if (isOuter) { + sb.append(Randomly.fromOptions("HASH", "MERGE", "LOOKUP")); + } else { + sb.append(Randomly.fromOptions("HASH", "MERGE")); + } sb.append(" "); } } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBAggregate.java b/src/sqlancer/cockroachdb/ast/CockroachDBAggregate.java index 0bddde8af..b7413b6d6 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBAggregate.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBAggregate.java @@ -78,10 +78,6 @@ public static List getAggregates(CockroachDBDataTy this.supportedReturnTypes = supportedReturnTypes.clone(); } - public static CockroachDBAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - public static CockroachDBAggregateFunction getRandomMetamorphicOracle() { // not: VARIANCE, STDDEV, SQRDIFF return Randomly.fromOptions(SUM, SUM_INT, MIN, MAX, XOR_AGG, BIT_AND, BIT_OR, BOOL_AND, BOOL_OR, COUNT, AVG, diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java b/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java index 9feb569af..3f083b5e3 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class CockroachDBAlias implements UnaryOperation, CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryArithmeticOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryArithmeticOperation.java index 0eb4da615..57e80dd61 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryArithmeticOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryArithmeticOperation.java @@ -1,9 +1,9 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; import sqlancer.cockroachdb.ast.CockroachDBBinaryArithmeticOperation.CockroachDBBinaryArithmeticOperator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class CockroachDBBinaryArithmeticOperation extends BinaryOperatorNode diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryComparisonOperator.java b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryComparisonOperator.java index 39ae85e7c..00a4ecf9c 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryComparisonOperator.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryComparisonOperator.java @@ -1,9 +1,9 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; import sqlancer.cockroachdb.ast.CockroachDBBinaryComparisonOperator.CockroachDBComparisonOperator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class CockroachDBBinaryComparisonOperator extends BinaryOperatorNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryLogicalOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryLogicalOperation.java index 352238afb..15695136a 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBBinaryLogicalOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBBinaryLogicalOperation.java @@ -1,9 +1,9 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation.CockroachDBBinaryLogicalOperator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public class CockroachDBBinaryLogicalOperation extends BinaryOperatorNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBCast.java b/src/sqlancer/cockroachdb/ast/CockroachDBCast.java index 79fb19929..d4b54e4da 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBCast.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBCast.java @@ -1,7 +1,7 @@ package sqlancer.cockroachdb.ast; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class CockroachDBCast implements UnaryOperation, CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBCollate.java b/src/sqlancer/cockroachdb/ast/CockroachDBCollate.java index fe10a8c76..55ec5c99b 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBCollate.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBCollate.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.UnaryNode; +import sqlancer.common.ast.UnaryNode; public class CockroachDBCollate extends UnaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBConcatOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBConcatOperation.java index 9cab969fa..bb71c6ab4 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBConcatOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBConcatOperation.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.BinaryNode; +import sqlancer.common.ast.BinaryNode; public class CockroachDBConcatOperation extends BinaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java b/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java index 84e2cb996..aa4dabcc4 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java @@ -230,20 +230,22 @@ public static CockroachDBExpression createBitConstantWithSize(int size) { } public static CockroachDBExpression createTimestampConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIMESTAMP", integer, "yyyy-MM-dd"); + return new CockroachDBTimeRelatedConstant("TIMESTAMP", integer, "yyyy-MM-dd'T'HH:mm:ss"); } public static CockroachDBExpression createTimeConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIME", integer, "HH:mm:ss"); + return new CockroachDBTimeRelatedConstant("TIME", integer, "yyyy-MM-dd'T'HH:mm:ss"); } public static CockroachDBExpression createTimetz(long integer) { - return new CockroachDBTimeRelatedConstant("TIMETZ", integer, "HH:mm:ss"); // TODO: support the complete format + return new CockroachDBTimeRelatedConstant("TIMETZ", integer, "yyyy-MM-dd'T'HH:mm:ss"); // TODO: support the + // complete format } public static CockroachDBExpression createTimestamptzConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIMESTAMPTZ", integer, "HH:mm:ss"); // TODO: support the complete - // format + return new CockroachDBTimeRelatedConstant("TIMESTAMPTZ", integer, "yyyy-MM-dd'T'HH:mm:ss"); // TODO: support the + // complete + // format } public static CockroachDBExpression createIntervalConstant(long year, long month, long day, long hour, long minute, diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java b/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java index 555fb97fd..d0ac07310 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.cockroachdb.ast; -public interface CockroachDBExpression { +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.common.ast.newast.Expression; + +public interface CockroachDBExpression extends Expression { } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBIndexReference.java b/src/sqlancer/cockroachdb/ast/CockroachDBIndexReference.java index e9410acce..d9f7f6874 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBIndexReference.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBIndexReference.java @@ -1,8 +1,8 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.schema.TableIndex; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.schema.TableIndex; +import sqlancer.common.visitor.UnaryOperation; public class CockroachDBIndexReference extends CockroachDBTableReference implements UnaryOperation { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java index 740a7901c..a8cba22f9 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java @@ -1,28 +1,31 @@ package sqlancer.cockroachdb.ast; +import java.util.Arrays; + import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.common.ast.newast.Join; -public class CockroachDBJoin implements CockroachDBExpression { +public class CockroachDBJoin + implements CockroachDBExpression, Join { private final CockroachDBExpression leftTable; private final CockroachDBExpression rightTable; - private final JoinType joinType; - private final CockroachDBExpression onCondition; - private OuterType outerType; + private JoinType joinType; + private CockroachDBExpression onCondition; public enum JoinType { - INNER, NATURAL, CROSS, OUTER; + INNER, LEFT, RIGHT, FULL, CROSS, NATURAL; public static JoinType getRandom() { return Randomly.fromOptions(values()); } - } - - public enum OuterType { - FULL, LEFT, RIGHT; - public static OuterType getRandom() { - return Randomly.fromOptions(values()); + public static JoinType getRandomExcept(JoinType... exclude) { + JoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(JoinType[]::new); + return Randomly.fromOptions(values); } } @@ -42,35 +45,29 @@ public CockroachDBExpression getRightTable() { return rightTable; } - public JoinType getJoinType() { - return joinType; - } - - public CockroachDBExpression getOnCondition() { - return onCondition; - } - - public static CockroachDBJoin createNaturalJoin(CockroachDBExpression left, CockroachDBExpression right) { - return new CockroachDBJoin(left, right, JoinType.NATURAL, null); + public void setJoinType(JoinType joinType) { + this.joinType = joinType; } - public static CockroachDBJoin createCrossJoin(CockroachDBExpression left, CockroachDBExpression right) { - return new CockroachDBJoin(left, right, JoinType.CROSS, null); + public JoinType getJoinType() { + return joinType; } - public static CockroachDBJoin createOuterJoin(CockroachDBExpression left, CockroachDBExpression right, - OuterType type, CockroachDBExpression onClause) { - CockroachDBJoin join = new CockroachDBJoin(left, right, JoinType.OUTER, onClause); - join.setOuterType(type); - return join; + @Override + public void setOnClause(CockroachDBExpression onCondition) { + this.onCondition = onCondition; } - private void setOuterType(OuterType outerType) { - this.outerType = outerType; + public CockroachDBExpression getOnCondition() { + return onCondition; } - public OuterType getOuterType() { - return outerType; + public static CockroachDBJoin createJoin(CockroachDBExpression left, CockroachDBExpression right, JoinType type, + CockroachDBExpression onClause) { + if (type.compareTo(JoinType.CROSS) >= 0) { + return new CockroachDBJoin(left, right, type, null); + } else { + return new CockroachDBJoin(left, right, type, onClause); + } } - } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBNotOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBNotOperation.java index 72f6d8e31..05ca2251e 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBNotOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBNotOperation.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.UnaryNode; +import sqlancer.common.ast.UnaryNode; public class CockroachDBNotOperation extends UnaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBOrderingTerm.java b/src/sqlancer/cockroachdb/ast/CockroachDBOrderingTerm.java index 1d23e9167..905bc7771 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBOrderingTerm.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBOrderingTerm.java @@ -1,6 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class CockroachDBOrderingTerm implements UnaryOperation, CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBRegexOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBRegexOperation.java index a4609b63e..24a75a92f 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBRegexOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBRegexOperation.java @@ -1,7 +1,7 @@ package sqlancer.cockroachdb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryNode; +import sqlancer.common.ast.BinaryNode; public class CockroachDBRegexOperation extends BinaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java index f8d6c1a1a..c043a9207 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.SelectBase; +import java.util.List; +import java.util.stream.Collectors; -public class CockroachDBSelect extends SelectBase implements CockroachDBExpression { +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; + +public class CockroachDBSelect extends SelectBase implements CockroachDBExpression, + Select { private boolean isDistinct; @@ -14,4 +22,21 @@ public void setDistinct(boolean isDistinct) { this.isDistinct = isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (CockroachDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (CockroachDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return CockroachDBVisitor.asString(this); + } + } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBTypeAnnotation.java b/src/sqlancer/cockroachdb/ast/CockroachDBTypeAnnotation.java index acab86dcb..b47d2147f 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBTypeAnnotation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBTypeAnnotation.java @@ -1,7 +1,7 @@ package sqlancer.cockroachdb.ast; -import sqlancer.ast.UnaryNode; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; +import sqlancer.common.ast.UnaryNode; public class CockroachDBTypeAnnotation extends UnaryNode implements CockroachDBExpression { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java index aadc17626..b880ea67d 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryArithmeticOperation.java @@ -1,9 +1,8 @@ package sqlancer.cockroachdb.ast; -import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; import sqlancer.cockroachdb.ast.CockroachDBUnaryArithmeticOperation.CockroachDBUnaryAritmeticOperator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; public class CockroachDBUnaryArithmeticOperation extends UnaryOperatorNode implements CockroachDBExpression { @@ -17,10 +16,6 @@ public enum CockroachDBUnaryAritmeticOperator implements Operator { this.textRepr = textRepr; } - public static CockroachDBUnaryAritmeticOperator getRandom() { - return Randomly.fromOptions(values()); - } - @Override public String getTextRepresentation() { return textRepr; diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java index 581a66088..0b28e4e88 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBUnaryPostfixOperation.java @@ -1,9 +1,8 @@ package sqlancer.cockroachdb.ast; -import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; public class CockroachDBUnaryPostfixOperation extends UnaryOperatorNode implements CockroachDBExpression { @@ -21,10 +20,6 @@ public enum CockroachDBUnaryPostfixOperator implements Operator { this.s = s; } - public static CockroachDBUnaryPostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - @Override public String getTextRepresentation() { return s; diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java index 9ee52a41c..a7d6f3a30 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCommentOnGenerator.java @@ -1,18 +1,16 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.schema.TableIndex; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.TableIndex; public final class CockroachDBCommentOnGenerator { @@ -23,12 +21,13 @@ private enum Option { TABLE, INDEX, COLUMN } - public static Query comment(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter comment(CockroachDBGlobalState globalState) { StringBuilder sb = new StringBuilder("COMMENT ON "); CockroachDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); switch (Randomly.fromOptions(Option.values())) { case TABLE: - sb.append("TABLE " + randomTable.getName()); + sb.append("TABLE "); + sb.append(randomTable.getName()); break; case INDEX: List indexes = randomTable.getIndexes(); @@ -39,7 +38,8 @@ public static Query comment(CockroachDBGlobalState globalState) { if (index.getIndexName().contains("primary")) { throw new IgnoreMeException(); } - sb.append("INDEX " + index.getIndexName()); + sb.append("INDEX "); + sb.append(index.getIndexName()); break; case COLUMN: sb.append("COLUMN "); @@ -53,9 +53,9 @@ public static Query comment(CockroachDBGlobalState globalState) { sb.append(" IS '"); sb.append(globalState.getRandomly().getString().replace("'", "''")); sb.append("'"); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java index 9def2dcbf..b8895a34e 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java @@ -1,19 +1,17 @@ package sqlancer.cockroachdb.gen; -import java.util.Arrays; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBCreateStatisticsGenerator { private CockroachDBCreateStatisticsGenerator() { } - public static Query create(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter create(CockroachDBGlobalState globalState) { CockroachDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); StringBuilder sb = new StringBuilder("CREATE STATISTICS s"); sb.append(Randomly.smallNumber()); @@ -24,12 +22,7 @@ public static Query create(CockroachDBGlobalState globalState) { sb.append(" FROM "); sb.append(randomTable.getName()); - return new QueryAdapter(sb.toString(), - Arrays.asList("current transaction is aborted, commands ignored until end of transaction block", - "ERROR: unable to encode table key: *tree.DArray" /* - * https://github.com/cockroachdb/cockroach/ - * issues/46964 - */, "overflow during Encode")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("overflow during Encode")); // https://github.com/cockroachdb/cockroach/issues/84078 } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index 3cae2cd19..dd1e94aa9 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -1,39 +1,39 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; + +public final class CockroachDBDeleteGenerator extends AbstractDeleteGenerator { -public final class CockroachDBDeleteGenerator { + private final CockroachDBGlobalState globalState; + + private CockroachDBDeleteGenerator(CockroachDBGlobalState globalState) { + this.globalState = globalState; + } - private CockroachDBDeleteGenerator() { + public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { + return new CockroachDBDeleteGenerator(globalState).getStatement(); } - public static Query delete(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); - StringBuilder sb = new StringBuilder(); + @Override + public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); CockroachDBErrors.addExpressionErrors(errors); - sb.append(CockroachDBVisitor.asString(new CockroachDBExpressionGenerator(globalState) + appendWhereClause(CockroachDBVisitor.asString(new CockroachDBExpressionGenerator(globalState) .setColumns(table.getColumns()).generateExpression(CockroachDBDataType.BOOL.get()))); } else { errors.add("rejected: DELETE without WHERE clause (sql_safe_updates = true)"); } errors.add("foreign key violation"); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java new file mode 100644 index 000000000..3834c8757 --- /dev/null +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java @@ -0,0 +1,35 @@ +package sqlancer.cockroachdb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class CockroachDBDropTableGenerator { + + private CockroachDBDropTableGenerator() { + } + + public static SQLQueryAdapter drop(CockroachDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + + ExpectedErrors errors = new ExpectedErrors(); + errors.add("is referenced by foreign key"); + + StringBuilder sb = new StringBuilder(); + sb.append("DROP"); + sb.append(" TABLE"); + sb.append(" "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), true); + } + +} diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java new file mode 100644 index 000000000..5d440b564 --- /dev/null +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.cockroachdb.gen; + +import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class CockroachDBDropViewGenerator { + + private CockroachDBDropViewGenerator() { + } + + public static SQLQueryAdapter drop(CockroachDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("is referenced by foreign key"); + + StringBuilder sb = new StringBuilder(); + sb.append("DROP"); + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + } + sb.append(" VIEW"); + sb.append(" "); + if (Randomly.getBooleanWithRatherLowProbability()) { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(globalState.getSchema().getRandomTable(t -> t.isView()).getName()); + } + } else { + sb.append(globalState.getSchema().getRandomTable(t -> t.isView()).getName()); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), true); + } + +} diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index bc5f765c2..8aeb5492b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -2,14 +2,18 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.cockroachdb.ast.CockroachDBAggregate; import sqlancer.cockroachdb.ast.CockroachDBAggregate.CockroachDBAggregateFunction; import sqlancer.cockroachdb.ast.CockroachDBBetweenOperation; @@ -29,6 +33,8 @@ import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBFunction; import sqlancer.cockroachdb.ast.CockroachDBInOperation; +import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBJoin.JoinType; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonOperator; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonType; @@ -36,14 +42,24 @@ import sqlancer.cockroachdb.ast.CockroachDBOrderingTerm; import sqlancer.cockroachdb.ast.CockroachDBRegexOperation; import sqlancer.cockroachdb.ast.CockroachDBRegexOperation.CockroachDBRegexOperator; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.ast.CockroachDBTypeAnnotation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; -import sqlancer.gen.TypedExpressionGenerator; +import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; -public class CockroachDBExpressionGenerator - extends TypedExpressionGenerator { +public class CockroachDBExpressionGenerator extends + TypedExpressionGenerator implements + NoRECGenerator, + TLPWhereGenerator, + CERTGenerator { + private List tables; private final CockroachDBGlobalState globalState; public CockroachDBExpressionGenerator(CockroachDBGlobalState globalState) { @@ -84,7 +100,8 @@ public List getOrderingTerms() { @Override public CockroachDBExpression generateExpression(CockroachDBCompositeDataType type, int depth) { - // if (type == CockroachDBDataType.FLOAT && Randomly.getBooleanWithRatherLowProbability()) { + // if (type == CockroachDBDataType.FLOAT && + // Randomly.getBooleanWithRatherLowProbability()) { // type = CockroachDBDataType.INT; // } if (allowAggregates && Randomly.getBoolean()) { @@ -343,4 +360,242 @@ protected CockroachDBExpression generateColumn(CockroachDBCompositeDataType type return columnReference; } + @Override + public CockroachDBExpression generatePredicate() { + return generateExpression(CockroachDBDataType.BOOL.get()); + } + + @Override + public CockroachDBExpression negatePredicate(CockroachDBExpression predicate) { + return new CockroachDBNotOperation(predicate); + } + + @Override + public CockroachDBExpression isNull(CockroachDBExpression expr) { + return new CockroachDBUnaryPostfixOperation(expr, CockroachDBUnaryPostfixOperator.IS_NULL); + } + + @Override + public CockroachDBExpressionGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public CockroachDBExpression generateBooleanExpression() { + return generateExpression(CockroachDBDataType.BOOL.get()); + } + + @Override + public CockroachDBSelect generateSelect() { + return new CockroachDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinExpressions = new ArrayList<>(); + List tableReferences = tables.stream().map(t -> new CockroachDBTableReference(t)) + .collect(Collectors.toList()); + while (tableReferences.size() >= 2 && Randomly.getBoolean()) { + CockroachDBTableReference leftTable = tableReferences.remove(0); + CockroachDBTableReference rightTable = tableReferences.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + + tables = tableReferences.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joinExpressions; + } + + @Override + public List getTableRefs() { + List tableReferences = tables.stream().map(t -> new CockroachDBTableReference(t)) + .collect(Collectors.toList()); + + return CockroachDBCommon.getTableReferences(tableReferences); + } + + @Override + public String generateOptimizedQueryString(CockroachDBSelect select, CockroachDBExpression whereCondition, + boolean shouldUseAggregate) { + CockroachDBColumn c = new CockroachDBColumn("COUNT(*)", null, false, false); + select.setWhereClause(whereCondition); + if (shouldUseAggregate) { + CockroachDBAggregate aggr = new CockroachDBAggregate(CockroachDBAggregateFunction.COUNT, + List.of(new CockroachDBColumnReference(new CockroachDBColumn("*", + new CockroachDBCompositeDataType(CockroachDBDataType.INT, 0), false, false)))); + select.setFetchColumns(List.of(aggr)); + } else { + select.setFetchColumns(List.of(new CockroachDBColumnReference(c))); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(getOrderingTerms()); + } + } + return CockroachDBVisitor.asString(select); + } + + @Override + public String generateUnoptimizedQueryString(CockroachDBSelect select, CockroachDBExpression whereCondition) { + List tableList = select.getFromList(); + List joinList = select.getJoinList(); + String fromString = tableList.stream().map(t -> ((CockroachDBTableReference) t).getTable().getName()) + .collect(Collectors.joining(", ")); + if (!tableList.isEmpty() && !joinList.isEmpty()) { + fromString += ", "; + } + return "SELECT SUM(count) FROM (SELECT CAST(" + CockroachDBVisitor.asString(whereCondition) + + " IS TRUE AS INT) as count FROM " + fromString + " " + + joinList.stream().map(j -> CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy || columns.isEmpty()) { + return List.of(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new CockroachDBColumnReference(c)) + .collect(Collectors.toList()); + } + + @Override + public String generateExplainQuery(CockroachDBSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(CockroachDBSelect select) { + List> mutators = new ArrayList<>(); + + if (!CockroachDBBugs.bug131647) { + mutators.add(this::mutateJoin); + } + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateAnd); + if (!CockroachDBBugs.bug131640) { + mutators.add(this::mutateWhere); + mutators.add(this::mutateOr); + } + // mutators.add(this::mutateLimit); + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateJoin(CockroachDBSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + + CockroachDBJoin join = (CockroachDBJoin) Randomly.fromList(select.getJoinList()); + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((CockroachDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((CockroachDBTableReference) join.getRightTable()).getTable().getColumns()); + CockroachDBExpressionGenerator joinGen2 = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + join.setOnClause(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); + } + + JoinType newJoinType = CockroachDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invariant relation + // between LEFT and RIGHT + // join + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS, JoinType.LEFT, + JoinType.RIGHT); + } else if (join.getJoinType() == JoinType.FULL) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS); + } else if (join.getJoinType() != JoinType.CROSS) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + return increase; + } + + boolean mutateDistinct(CockroachDBSelect select) { + boolean increase = select.isDistinct(); + select.setDistinct(!select.isDistinct()); + return increase; + } + + boolean mutateWhere(CockroachDBSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + } + return increase; + } + + boolean mutateGroupBy(CockroachDBSelect select) { + boolean increase = !select.getGroupByExpressions().isEmpty(); + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(CockroachDBSelect select) { + if (select.getGroupByExpressions().isEmpty()) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(CockroachDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(CockroachDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(CockroachDBSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(generateConstant(CockroachDBDataType.INT.get())); + } + return increase; + } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java index fe0e0fb55..34d0b9355 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java @@ -1,13 +1,11 @@ package sqlancer.cockroachdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.gen.AbstractGenerator; +import sqlancer.common.gen.AbstractGenerator; public abstract class CockroachDBGenerator extends AbstractGenerator { @@ -32,21 +30,4 @@ static void addColumns(StringBuilder sb, List columns, boolea sb.append(")"); } - void generateInterleave() { - // TODO make this more likely to succeed - CockroachDBTable parentTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - List parentColumns = parentTable.getRandomNonEmptyColumnSubset(); - sb.append(" INTERLEAVE IN PARENT "); - sb.append(parentTable.getName()); - sb.append("("); - sb.append(parentColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - errors.add("must refer to a prefix of the primary key column names being interleaved"); - errors.add("must refer to a prefix of the index column names being interleaved"); - errors.add("must match the parent's primary index"); - errors.add("must match type and sort direction of the parent's primary index"); - errors.add("must be a prefix of the index columns being interleaved"); - errors.add("must be a prefix of the primary key columns being interleaved"); - } - } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 4aae1ce78..da6d3c609 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -2,11 +2,13 @@ import java.util.List; -import sqlancer.Query; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.common.query.SQLQueryAdapter; // https://www.cockroachlabs.com/docs/stable/create-index.html public class CockroachDBIndexGenerator extends CockroachDBGenerator { @@ -15,8 +17,11 @@ public CockroachDBIndexGenerator(CockroachDBGlobalState globalState) { super(globalState); } - public static Query create(CockroachDBGlobalState s) { - return new CockroachDBIndexGenerator(s).getQuery(); + public static SQLQueryAdapter create(CockroachDBGlobalState s) { + if (s.getSchema().getIndexCount() >= s.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } + return new CockroachDBIndexGenerator(s).getStatement(); } @Override @@ -26,8 +31,14 @@ public void buildStatement() { errors.add("already contains column"); errors.add("violates unique constraint"); errors.add("schema change statement cannot follow a statement that has written in the same transaction"); - errors.add("https://github.com/cockroachdb/cockroach/issues/35730"); // some array types are not indexable + errors.add("and thus is not indexable"); // array types are not indexable + errors.add("the following columns are not indexable due to their type"); // array types are not indexable errors.add("cannot determine type of empty array. Consider annotating with the desired type"); + errors.add("incompatible IF expression"); // TODO: investigate; seems to be a bug + if (CockroachDBBugs.bug84154) { + errors.add("overflow during Encode"); + errors.add("of type interval"); + } CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append("CREATE "); if (Randomly.getBoolean()) { @@ -37,22 +48,23 @@ public void buildStatement() { sb.append(table.getName()); List columns = table.getRandomNonEmptyColumnSubset(); addColumns(sb, columns, true); - boolean hashSharded = globalState.getDmbsSpecificOptions().testHashIndexes + boolean hashSharded = globalState.getDbmsSpecificOptions().testHashIndexes && Randomly.getBooleanWithSmallProbability(); if (hashSharded) { sb.append(" USING HASH WITH BUCKET_COUNT="); - sb.append(Randomly.getNotCachedInteger(2, Short.MAX_VALUE)); + sb.append(Randomly.getNotCachedInteger(2, 2048)); errors.add("null value in column"); errors.add("cannot create a sharded index on a computed column"); } if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("STORING", "COVERING")); - sb.append(" "); - addColumns(sb, table.getRandomNonEmptyColumnSubset(), false); - } - if (!hashSharded /* interleaved indexes cannot also be hash sharded */ && Randomly.getBoolean()) { - generateInterleave(); + List columns2 = table.getRandomNonEmptyColumnSubset(); + columns2.removeAll(columns); + if (!columns2.isEmpty()) { + sb.append(" "); + sb.append(Randomly.fromOptions("STORING", "COVERING")); + sb.append(" "); + addColumns(sb, columns2, false); + } } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index 5cd9cf43e..9f823c59b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -1,26 +1,29 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBInsertGenerator { private CockroachDBInsertGenerator() { } - public static Query insert(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); + public static SQLQueryAdapter insert(CockroachDBGlobalState globalState) { + CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + return insert(globalState, table); + } + + public static SQLQueryAdapter insert(CockroachDBGlobalState globalState, CockroachDBTable table) { + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); // e.g., caused by computed columns errors.add("violates not-null constraint"); @@ -34,7 +37,6 @@ public static Query insert(CockroachDBGlobalState globalState) { errors.add("foreign key violation"); errors.add("multi-part foreign key"); StringBuilder sb = new StringBuilder(); - CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean isUpsert = Randomly.getBoolean(); if (!isUpsert) { sb.append("INSERT INTO "); @@ -79,8 +81,6 @@ public static Query insert(CockroachDBGlobalState globalState) { if (Randomly.getBoolean()) { sb.append(" NOTHING "); } else { - // TODO: also support excluded. (see - // https://www.cockroachlabs.com/docs/stable/insert.html) sb.append(" UPDATE SET "); List columns = table.getRandomNonEmptyColumnSubset(); int i = 0; @@ -90,14 +90,19 @@ public static Query insert(CockroachDBGlobalState globalState) { } sb.append(c.getName()); sb.append(" = "); - sb.append(CockroachDBVisitor.asString(gen.generateConstant(c.getType()))); + if (Randomly.getBoolean()) { + sb.append(CockroachDBVisitor.asString(gen.generateConstant(c.getType()))); + } else { + sb.append("excluded."); + sb.append(c.getName()); + } } errors.add("UPSERT or INSERT...ON CONFLICT command cannot affect row a second time"); } errors.add("there is no unique or exclusion constraint matching the ON CONFLICT specification"); } CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java index 572b9abf3..880f33310 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java @@ -4,8 +4,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -15,16 +13,17 @@ import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPBase; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBRandomQuerySynthesizer { private CockroachDBRandomQuerySynthesizer() { } - public static Query generate(CockroachDBGlobalState globalState, int nrColumns) { + public static SQLQueryAdapter generate(CockroachDBGlobalState globalState, int nrColumns) { CockroachDBSelect select = generateSelect(globalState, nrColumns); - return new QueryAdapter(CockroachDBVisitor.asString(select)); + return new SQLQueryAdapter(CockroachDBVisitor.asString(select)); } public static CockroachDBSelect generateSelect(CockroachDBGlobalState globalState, int nrColumns) { @@ -50,14 +49,14 @@ public static CockroachDBSelect generateSelect(CockroachDBGlobalState globalStat .map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList()); List updatedTableList = CockroachDBCommon.getTableReferences(tableList); if (Randomly.getBoolean()) { - select.setJoinList(CockroachDBNoRECOracle.getJoins(updatedTableList, globalState)); + select.setJoinList(CockroachDBTLPBase.getJoins(updatedTableList, globalState)); } select.setFromList(updatedTableList); if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 1d83d75f3..a948a1237 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -1,14 +1,12 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; import java.util.function.Function; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBSetClusterSettingGenerator { @@ -17,20 +15,14 @@ private CockroachDBSetClusterSettingGenerator() { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBClusterSetting { - COMPATOR_ENABLED("compactor.enabled", CockroachDBSetSessionGenerator::onOff), // - BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), // BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", - (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), // - RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), // - SQL_DEFAULTS_VECTORIZE_ROW_COUNT_THRESHOLD("sql.defaults.vectorize_row_count_threshold", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), - // SQL_DEFAULTS_EXPERIMENTAL_OPTIMIZER_FOREIGN_KEYS_ENABLED("sql.defaults.experimental_optimizer_foreign_keys.enabled", - // CockroachDBSetSessionGenerator::onOff), - MERGE_JOINS_ENABLED("sql.distsql.merge_joins.enabled", CockroachDBSetSessionGenerator::onOff), // - PARALLEL_SCANS_ENABLED("sql.parallel_scans.enabled", CockroachDBSetSessionGenerator::onOff), // - SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), // + RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", + (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), + SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), SQL_STATS_HISTOGRAM_COLLECTION_ENABLED("sql.stats.histogram_collection.enabled", - CockroachDBSetSessionGenerator::onOff); + CockroachDBSetSessionGenerator::onOff), + HISTOGRAM_COLLECT("sql.stats.histogram_collection.enabled", CockroachDBSetSessionGenerator::onOff); private Function f; private String name; @@ -41,7 +33,7 @@ private enum CockroachDBClusterSetting { } } - public static Query create(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter create(CockroachDBGlobalState globalState) { CockroachDBClusterSetting s = Randomly.fromOptions(CockroachDBClusterSetting.values()); StringBuilder sb = new StringBuilder("SET CLUSTER SETTING "); sb.append(s.name); @@ -51,12 +43,12 @@ public static Query create(CockroachDBGlobalState globalState) { } else { sb.append(s.f.apply(globalState)); } - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addTransactionErrors(errors); errors.add("setting updated but timed out waiting to read new value"); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index 341db2c42..0ade4a3ef 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -1,14 +1,12 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; import java.util.function.Function; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBSetSessionGenerator { @@ -21,23 +19,22 @@ public static String onOff(CockroachDBGlobalState globalState) { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBSetting { - BYTEA_OUTPUT((g) -> Randomly.fromOptions("hex", "escape", "base64")), // - DEFAULT_INT_SIZE((g) -> Randomly.fromOptions(4, 8)), // - DISTSQL((g) -> Randomly.fromOptions("on", "off", "auto", "always")), // - ENABLE_IMPLICIT_SELECT_FOR_UPDATE(CockroachDBSetSessionGenerator::onOff), // - ENABLE_INSERT_FAST_PATH(CockroachDBSetSessionGenerator::onOff), // + BYTEA_OUTPUT((g) -> Randomly.fromOptions("hex", "escape", "base64")), + DEFAULT_INT_SIZE((g) -> Randomly.fromOptions(4, 8)), + DISTSQL((g) -> Randomly.fromOptions("on", "off", "auto", "always")), + ENABLE_IMPLICIT_SELECT_FOR_UPDATE(CockroachDBSetSessionGenerator::onOff), + ENABLE_INSERT_FAST_PATH(CockroachDBSetSessionGenerator::onOff), ENABLE_ZIGZAG_JOIN(CockroachDBSetSessionGenerator::onOff), - // EXPERIMENTAL_ENABLE_HASH_SHARDED_INDEXES(CockroachDBSetSessionGenerator::onOff), - EXPERIMENTAL_SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), // - EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), // + SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), REORDER_JOINS_LIMIT((g) -> g.getRandomly().getInteger(0, Integer.MAX_VALUE)), // - SQL_SAFE_UPDATES(CockroachDBSetSessionGenerator::onOff), - // TRACING(CockroachDBSetSessionGenerator::onOff) - VECTORIZE((g) -> Randomly.fromOptions("auto", "on", - "off")); /* - * see https://github.com/cockroachdb/cockroach/issues/44133, - * https://github.com/cockroachdb/cockroach/issues/44207 - */ + SQL_SAFE_UPDATES((g) -> "off"), TRACING(CockroachDBSetSessionGenerator::onOff), + /* + * CockroachDB enables vectorized (column-oriented) execution by default. Row-oriented execution can be enforced + * by setting vectorized to "off". Some examples of bugs found in the vectorized execution engine are: + * https://github.com/cockroachdb/cockroach/issues/44133 https://github.com/cockroachdb/cockroach/issues/44207 + * + */ + VECTORIZE((g) -> Randomly.fromOptions("on", "off")); private Function f; @@ -46,15 +43,15 @@ private enum CockroachDBSetting { } } - public static Query create(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter create(CockroachDBGlobalState globalState) { CockroachDBSetting s = Randomly.fromOptions(CockroachDBSetting.values()); StringBuilder sb = new StringBuilder("SET SESSION "); sb.append(s); sb.append("="); sb.append(s.f.apply(globalState)); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java index 9414e2999..9b84d9c94 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java @@ -1,13 +1,10 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBShowGenerator { @@ -19,14 +16,14 @@ private enum Option { DATABASES, JOBS, RANGES, LOCALITY, SEQUENCES, TRACE_FOR_SESSION } - public static Query show(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); + public static SQLQueryAdapter show(CockroachDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); switch (Randomly.fromOptions(Option.values())) { case EXPERIMENTAL_FINGERPRINTS: - sb.append("SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE " - + globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); - errors.add("as type bytes: bytea encoded value ends with incomplete escape sequence"); + sb.append("SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + errors.add("bytea encoded value ends with incomplete escape sequence"); errors.add("invalid bytea escape sequence"); break; case DATABASES: @@ -36,7 +33,8 @@ public static Query show(CockroachDBGlobalState globalState) { sb.append("SHOW JOBS"); break; case RANGES: - sb.append("SHOW RANGES FROM TABLE " + globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + sb.append("SHOW RANGES FROM TABLE "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); break; case LOCALITY: sb.append("SHOW LOCALITY"); @@ -58,7 +56,7 @@ public static Query show(CockroachDBGlobalState globalState) { throw new AssertionError(); } CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index f654fd7ee..8678aff4d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -5,7 +5,6 @@ import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; @@ -15,6 +14,7 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.query.SQLQueryAdapter; public class CockroachDBTableGenerator extends CockroachDBGenerator { @@ -28,20 +28,24 @@ public CockroachDBTableGenerator(CockroachDBGlobalState globalState) { canAffectSchema = true; } - public static Query generate(CockroachDBGlobalState globalState) { - return new CockroachDBTableGenerator(globalState).getQuery(); + public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return new CockroachDBTableGenerator(globalState).getStatement(); } @Override public void buildStatement() { - errors.add("https://github.com/cockroachdb/cockroach/issues/35730"); // not indexable array types - if (globalState.getDmbsSpecificOptions().testTempTables) { + errors.add("and thus is not indexable"); // array types are not indexable + errors.add("context-dependent operators are not allowed in STORED COMPUTED COLUMN"); + if (globalState.getDbmsSpecificOptions().testTempTables) { errors.add("constraints on temporary tables may reference only temporary tables"); errors.add("constraints on permanent tables may reference only permanent tables"); } String tableName = globalState.getSchema().getFreeTableName(); sb.append("CREATE "); - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().testTempTables) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testTempTables) { sb.append("TEMP "); } sb.append("TABLE "); @@ -72,9 +76,16 @@ public void buildStatement() { && cockroachDBColumn.getType().getPrimitiveDataType() != CockroachDBDataType.SERIAL; if (generatedColumn) { sb.append(" AS ("); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(cockroachDBColumn.getType()))); + // To generate an expression exclude of the current column + List generatedColumns = new ArrayList<>(columns); + generatedColumns.remove(i); + CockroachDBExpressionGenerator genGeneratedColumn = new CockroachDBExpressionGenerator(globalState) + .setColumns(generatedColumns); + sb.append(CockroachDBVisitor + .asString(genGeneratedColumn.generateExpression(cockroachDBColumn.getType()))); sb.append(") STORED"); errors.add("computed columns cannot reference other computed columns"); + errors.add("context-dependent operators are not allowed in computed column"); errors.add("has type unknown"); } if (Randomly.getBooleanWithRatherLowProbability()) { @@ -143,32 +154,7 @@ public void buildStatement() { sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(")"); } - if (Randomly.getBoolean() && !globalState.getSchema().getDatabaseTables().isEmpty()) { - sb.append(", "); - // TODO: also allow referencing itself - List subset = Randomly.nonEmptySubset(columns); - sb.append(" FOREIGN KEY ("); - sb.append(subset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(") REFERENCES "); - CockroachDBTable otherTable = globalState.getSchema().getRandomTable(); - sb.append(otherTable.getName()); - sb.append("("); - for (int i = 0; i < subset.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(otherTable.getRandomColumn().getName()); - } - sb.append(")"); - // TODO: ensure that the column types match - errors.add("does not match foreign key"); - errors.add("computed column"); - errors.add("there is no unique constraint matching given keys for referenced table"); - } sb.append(")"); - if (Randomly.getBooleanWithRatherLowProbability() && !globalState.getSchema().getDatabaseTables().isEmpty()) { - generateInterleave(); - } errors.add("collatedstring"); CockroachDBErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index d8ec86ba9..44979487b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -1,12 +1,10 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBTruncateGenerator { @@ -14,13 +12,13 @@ private CockroachDBTruncateGenerator() { } // https://www.cockroachlabs.com/docs/v19.2/truncate.html - public static Query truncate(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); - errors.add("is interleaved by table"); + public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); errors.add("is referenced by foreign key"); + if (CockroachDBBugs.bug85230) { + errors.add("found in depended-on-by references, no such index in this relation"); + } - // https://github.com/cockroachdb/cockroach/issues/47030 - errors.add("unexpected value: "); StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); if (Randomly.getBoolean()) { @@ -41,7 +39,7 @@ public static Query truncate(CockroachDBGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index 5adcf62f1..06cc0f8ee 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -1,11 +1,7 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -13,18 +9,28 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; -public final class CockroachDBUpdateGenerator { +public final class CockroachDBUpdateGenerator extends AbstractUpdateGenerator { - private CockroachDBUpdateGenerator() { + private final CockroachDBGlobalState globalState; + private CockroachDBExpressionGenerator gen; + + private CockroachDBUpdateGenerator(CockroachDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { + return new CockroachDBUpdateGenerator(globalState).getStatement(); } - public static Query gen(CockroachDBGlobalState globalState) { - Set errors = new HashSet<>(); + @Override + public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); - StringBuilder sb = new StringBuilder("UPDATE "); + gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); + sb.append("UPDATE "); sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append("@{FORCE_INDEX="); @@ -32,18 +38,9 @@ public static Query gen(CockroachDBGlobalState globalState) { sb.append("}"); } sb.append(" SET "); - int i = 0; - for (CockroachDBColumn c : columns) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(c.getName()); - sb.append("="); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(c.getType()))); - } + updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); + appendWhereClause(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); } errors.add("violates unique constraint"); errors.add("violates not-null constraint"); @@ -54,7 +51,11 @@ public static Query gen(CockroachDBGlobalState globalState) { errors.add("cannot write directly to computed column"); CockroachDBErrors.addExpressionErrors(errors); CockroachDBErrors.addTransactionErrors(errors); - return new QueryAdapter(sb.toString(), errors); + } + + @Override + protected void updateValue(CockroachDBColumn column) { + sb.append(CockroachDBVisitor.asString(gen.generateExpression(column.getType()))); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java index d73b7b9fd..8754f54d1 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBViewGenerator.java @@ -1,20 +1,17 @@ package sqlancer.cockroachdb.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBViewGenerator { private CockroachDBViewGenerator() { } - public static Query generate(CockroachDBGlobalState globalState) { + public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); @@ -24,16 +21,17 @@ public static Query generate(CockroachDBGlobalState globalState) { if (i != 0) { sb.append(", "); } - sb.append("c" + i); + sb.append("c"); + sb.append(i); } sb.append(") AS "); sb.append(CockroachDBRandomQuerySynthesizer.generate(globalState, nrColumns).getQueryString()); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); CockroachDBErrors.addTransactionErrors(errors); errors.add("value type unknown cannot be used for table columns"); errors.add("already exists"); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java new file mode 100644 index 000000000..7d2af5930 --- /dev/null +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -0,0 +1,292 @@ +package sqlancer.cockroachdb.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.cockroachdb.CockroachDBBugs; +import sqlancer.cockroachdb.CockroachDBCommon; +import sqlancer.cockroachdb.CockroachDBErrors; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; +import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation; +import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation.CockroachDBBinaryLogicalOperator; +import sqlancer.cockroachdb.ast.CockroachDBColumnReference; +import sqlancer.cockroachdb.ast.CockroachDBExpression; +import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBJoin.JoinType; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableReference; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + +public class CockroachDBCERTOracle extends CERTOracleBase + implements TestOracle { + private CockroachDBExpressionGenerator gen; + private CockroachDBSelect select; + + public CockroachDBCERTOracle(CockroachDBGlobalState globalState) { + super(globalState); + CockroachDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Randomly generate a query + CockroachDBTables tables = state.getSchema().getRandomTableNonEmptyTables(2); + List tableList = CockroachDBCommon.getTableReferences( + tables.getTables().stream().map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList())); + gen = new CockroachDBExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new CockroachDBColumnReference(c)).collect(Collectors.toList())); + select = new CockroachDBSelect(); + select.setFetchColumns(fetchColumns); + select.setFromList(tableList); + select.setDistinct(Randomly.getBoolean()); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + } + + // Set the join. + List joinExpressions = getJoins(tableList, state); + select.setJoinList(joinExpressions); + + // Get the result of the first query + String queryString1 = CockroachDBVisitor.asString(select); + int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + List excludes = new ArrayList<>(); + // Disable limit due to its false positive + excludes.add(Mutator.LIMIT); + if (CockroachDBBugs.bug131640) { + excludes.add(Mutator.OR); + } + if (CockroachDBBugs.bug131647) { + excludes.add(Mutator.JOIN); + } + // Mutate the query + boolean increase = mutate(excludes.toArray(new Mutator[0])); + + // Get the result of the second query + String queryString2 = CockroachDBVisitor.asString(select); + int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check structural equivalence + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check the results + if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + private List getJoins(List tableList, + CockroachDBGlobalState globalState) throws AssertionError { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getPercentage() < 0.8) { + CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); + CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, + CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + return joinExpressions; + } + + @Override + protected boolean mutateJoin() { + if (select.getJoinList().isEmpty()) { + return false; + } + + CockroachDBJoin join = (CockroachDBJoin) Randomly.fromList(select.getJoinList()); + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((CockroachDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((CockroachDBTableReference) join.getRightTable()).getTable().getColumns()); + CockroachDBExpressionGenerator joinGen2 = new CockroachDBExpressionGenerator(state).setColumns(columns); + join.setOnClause(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); + } + + JoinType newJoinType = CockroachDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invariant relation + // between LEFT and RIGHT + // join + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS, JoinType.LEFT, + JoinType.RIGHT); + } else if (join.getJoinType() == JoinType.FULL) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS); + } else if (join.getJoinType() != JoinType.CROSS) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + return increase; + } + + @Override + protected boolean mutateDistinct() { + boolean increase = select.isDistinct(); + select.setDistinct(!select.isDistinct()); + return increase; + } + + @Override + protected boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + return increase; + } + + @Override + protected boolean mutateGroupBy() { + boolean increase = !select.getGroupByExpressions().isEmpty(); + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + @Override + protected boolean mutateHaving() { + if (select.getGroupByExpressions().isEmpty()) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + @Override + protected boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + @Override + protected boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + @Override + protected boolean mutateLimit() { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(gen.generateConstant(CockroachDBDataType.INT.get())); + } + return increase; + } + + private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + int row = -1; + String explainQuery = "EXPLAIN " + selectStr; + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String content = rs.getString(1); + if (content.contains("count:")) { + try { + int number = Integer.parseInt(content.split("count: ")[1].split(" ")[0].replace(",", "")); + if (row == -1) { + row = number; + } + } catch (Exception e) { // To avoid the situation that no number is found + } + } + if (content.contains("• ")) { + String operation = content.split("• ")[1].split(" ")[0]; + if (CockroachDBBugs.bug131875 && (operation.equals("distinct") || operation.equals("limit"))) { + throw new IgnoreMeException(); + } + queryPlanSequences.add(operation); + } + } + } + } catch (IgnoreMeException e) { + throw new IgnoreMeException(); + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + } + +} diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java deleted file mode 100644 index 3babf7ee6..000000000 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ /dev/null @@ -1,158 +0,0 @@ -package sqlancer.cockroachdb.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.Randomly; -import sqlancer.TestOracle; -import sqlancer.cockroachdb.CockroachDBCommon; -import sqlancer.cockroachdb.CockroachDBErrors; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; -import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.cockroachdb.ast.CockroachDBColumnReference; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBJoin.OuterType; -import sqlancer.cockroachdb.ast.CockroachDBSelect; -import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; - -public class CockroachDBNoRECOracle implements TestOracle { - - private final CockroachDBGlobalState globalState; - private final Set errors = new HashSet<>(); - private String optimizableQueryString; - private String unoptimizedQuery; - private CockroachDBExpressionGenerator gen; - - public CockroachDBNoRECOracle(CockroachDBGlobalState globalState) { - this.globalState = globalState; - CockroachDBErrors.addExpressionErrors(errors); - CockroachDBErrors.addTransactionErrors(errors); - errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; - errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; - - } - - @Override - public void check() throws SQLException { - CockroachDBTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); - List tableL = tables.getTables().stream().map(t -> new CockroachDBTableReference(t)) - .collect(Collectors.toList()); - List tableList = CockroachDBCommon.getTableReferences(tableL); - gen = new CockroachDBExpressionGenerator(globalState).setColumns(tables.getColumns()); - List joinExpressions = getJoins(tableList, globalState); - CockroachDBExpression whereCondition = gen.generateExpression(CockroachDBDataType.BOOL.get()); - int optimizableCount = getOptimizedResult(whereCondition, tableList, errors, joinExpressions); - if (optimizableCount == -1) { - throw new IgnoreMeException(); - } - int nonOptimizableCount = getNonOptimizedResult(whereCondition, tableList, errors, joinExpressions); - if (nonOptimizableCount == -1) { - throw new IgnoreMeException(); - } - if (optimizableCount != nonOptimizableCount) { - globalState.getState().queryString = optimizableQueryString + ";\n" + unoptimizedQuery + ";"; - throw new AssertionError(CockroachDBVisitor.asString(whereCondition)); - } - } - - public static List getJoins(List tableList, - CockroachDBGlobalState globalState) throws AssertionError { - List joinExpressions = new ArrayList<>(); - while (tableList.size() >= 2 && Randomly.getBoolean()) { - CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); - List columns = new ArrayList<>(leftTable.getTable().getColumns()); - columns.addAll(rightTable.getTable().getColumns()); - CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) - .setColumns(columns); - switch (CockroachDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(new CockroachDBJoin(leftTable, rightTable, CockroachDBJoin.JoinType.INNER, - joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); - break; - case NATURAL: - joinExpressions.add(CockroachDBJoin.createNaturalJoin(leftTable, rightTable)); - break; - case CROSS: - joinExpressions.add(CockroachDBJoin.createCrossJoin(leftTable, rightTable)); - break; - case OUTER: - joinExpressions.add(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.getRandom(), - joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); - break; - default: - throw new AssertionError(); - } - } - return joinExpressions; - } - - private int getOptimizedResult(CockroachDBExpression whereCondition, List tableList, - Set errors, List joinExpressions) throws SQLException { - CockroachDBSelect select = new CockroachDBSelect(); - CockroachDBColumn c = new CockroachDBColumn("COUNT(*)", null, false, false); - select.setFetchColumns(Arrays.asList(new CockroachDBColumnReference(c))); - select.setFromList(tableList); - select.setWhereClause(whereCondition); - select.setJoinList(joinExpressions); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.getOrderingTerms()); - } - String s = CockroachDBVisitor.asString(select); - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(s); - } - this.optimizableQueryString = s; - Query q = new QueryAdapter(s, errors); - return getCount(globalState, q); - } - - private int getNonOptimizedResult(CockroachDBExpression whereCondition, List tableList, - Set errors, List joinList) throws SQLException { - String fromString = tableList.stream().map(t -> ((CockroachDBTableReference) t).getTable().getName()) - .collect(Collectors.joining(", ")); - if (!tableList.isEmpty() && !joinList.isEmpty()) { - fromString += ", "; - } - String s = "SELECT SUM(count) FROM (SELECT CAST(" + CockroachDBVisitor.asString(whereCondition) - + " IS TRUE AS INT) as count FROM " + fromString + " " - + joinList.stream().map(j -> CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(s); - } - this.unoptimizedQuery = s; - Query q = new QueryAdapter(s, errors); - return getCount(globalState, q); - } - - private int getCount(GlobalState globalState, Query q) throws AssertionError { - int count = 0; - try (ResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return -1; - } - if (rs.next()) { - count = rs.getInt(1); - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - return count; - } - -} diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index c6b531ccd..6dd4bd5d5 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -1,21 +1,16 @@ package sqlancer.cockroachdb.oracle.tlp; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import org.postgresql.util.PSQLException; import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -34,12 +29,15 @@ import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; -public class CockroachDBTLPAggregateOracle implements TestOracle { +public class CockroachDBTLPAggregateOracle implements TestOracle { private final CockroachDBGlobalState state; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private CockroachDBExpressionGenerator gen; private String firstResult; private String secondResult; @@ -73,21 +71,21 @@ public void check() throws SQLException { .map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList()); List from = CockroachDBCommon.getTableReferences(tableList); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setJoinList(CockroachDBNoRECOracle.getJoins(from, state)); + select.setJoinList(CockroachDBTLPBase.getJoins(from, state)); } select.setFromList(from); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } originalQuery = CockroachDBVisitor.asString(select); firstResult = getAggregateResult(originalQuery); metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, from); secondResult = getAggregateResult(metamorphicQuery); - state.getState().queryString = "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult - + "\n-- " + secondResult; - if (firstResult == null && secondResult != null - || firstResult != null && (!firstResult.contentEquals(secondResult) + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && secondResult != null && (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { if (secondResult.contains("Inf")) { throw new IgnoreMeException(); // FIXME: average computation @@ -108,7 +106,7 @@ private String createMetamorphicUnionQuery(CockroachDBSelect select, CockroachDB CockroachDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); CockroachDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); CockroachDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate).toString() + " FROM ("; + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; metamorphicQuery += CockroachDBVisitor.asString(leftSelect) + " UNION ALL " + CockroachDBVisitor.asString(middleSelect) + " UNION ALL " + CockroachDBVisitor.asString(rightSelect); metamorphicQuery += ")"; @@ -117,8 +115,8 @@ private String createMetamorphicUnionQuery(CockroachDBSelect select, CockroachDB private String getAggregateResult(String queryString) throws SQLException { String resultString; - QueryAdapter q = new QueryAdapter(queryString, errors); - try (ResultSet result = q.executeAndGet(state)) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } @@ -148,13 +146,15 @@ private List mapped(CockroachDBAggregate aggregate) { case MIN: return aliasArgs(Arrays.asList(aggregate)); case AVG: - // List arg = Arrays.asList(new CockroachDBCast(aggregate.getExpr().get(0), + // List arg = Arrays.asList(new + // CockroachDBCast(aggregate.getExpr().get(0), // CockroachDBDataType.DECIMAL.get())); CockroachDBAggregate sum = new CockroachDBAggregate(CockroachDBAggregateFunction.SUM, aggregate.getExpr()); CockroachDBCast count = new CockroachDBCast( new CockroachDBAggregate(CockroachDBAggregateFunction.COUNT, aggregate.getExpr()), CockroachDBDataType.DECIMAL.get()); - // CockroachDBBinaryArithmeticOperation avg = new CockroachDBBinaryArithmeticOperation(sum, count, + // CockroachDBBinaryArithmeticOperation avg = new + // CockroachDBBinaryArithmeticOperation(sum, count, // CockroachDBBinaryArithmeticOperator.DIV); return aliasArgs(Arrays.asList(sum, count)); default: @@ -196,4 +196,9 @@ private CockroachDBSelect getSelect(List aggregates, List return leftSelect; } + @Override + public String getLastQueryString() { + return originalQuery; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index d141a68d9..d84cab56f 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -2,13 +2,10 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TestOracle; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema; @@ -18,29 +15,25 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; import sqlancer.cockroachdb.ast.CockroachDBColumnReference; import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBNotOperation; +import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; -public class CockroachDBTLPBase implements TestOracle { - - final CockroachDBGlobalState state; - final Set errors = new HashSet<>(); +public class CockroachDBTLPBase + extends TernaryLogicPartitioningOracleBase + implements TestOracle { CockroachDBSchema s; CockroachDBTables targetTables; CockroachDBExpressionGenerator gen; CockroachDBSelect select; - CockroachDBExpression predicate; - CockroachDBExpression negatedPredicate; - CockroachDBExpression isNullPredicate; public CockroachDBTLPBase(CockroachDBGlobalState state) { - this.state = state; + super(state); CockroachDBErrors.addExpressionErrors(errors); } @@ -49,23 +42,21 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new CockroachDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new CockroachDBSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new CockroachDBTableReference(t)) .collect(Collectors.toList()); - List joins = CockroachDBNoRECOracle.getJoins(tableList, state); + List joins = getJoins(tableList, state); select.setJoinList(joins); select.setFromList(tableList); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new CockroachDBNotOperation(predicate); - isNullPredicate = new CockroachDBUnaryPostfixOperation(predicate, CockroachDBUnaryPostfixOperator.IS_NULL); } List generateFetchColumns() { List columns = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() || targetTables.getColumns().isEmpty()) { columns.add(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); } else { columns.addAll(Randomly.nonEmptySubset(targetTables.getColumns()).stream() @@ -74,8 +65,25 @@ List generateFetchColumns() { return columns; } - CockroachDBExpression generatePredicate() { - return gen.generateExpression(CockroachDBDataType.BOOL.get()); + @Override + protected ExpressionGenerator getGen() { + return gen; + } + + public static List getJoins(List tableList, + CockroachDBGlobalState globalState) throws AssertionError { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); + CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + return joinExpressions; } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java index bdf8f59c3..dd174e805 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java @@ -15,6 +15,8 @@ public class CockroachDBTLPDistinctOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPDistinctOracle(CockroachDBGlobalState state) { super(state); errors.add("GROUP BY term out of range"); @@ -25,7 +27,7 @@ public void check() throws SQLException { super.check(); select.setDistinct(true); String originalQueryString = CockroachDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setDistinct(false); CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); @@ -41,4 +43,9 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java index 36ac46733..863f056ab 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java @@ -19,6 +19,7 @@ public class CockroachDBTLPExtendedWhereOracle extends CockroachDBTLPBase { private CockroachDBExpression originalPredicate; + private String generatedQueryString; public CockroachDBTLPExtendedWhereOracle(CockroachDBGlobalState state) { super(state); @@ -32,11 +33,12 @@ public void check() throws SQLException { originalPredicate = generatePredicate(); select.setWhereClause(originalPredicate); String originalQueryString = CockroachDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean allowOrderBy = Randomly.getBoolean(); if (allowOrderBy) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } select.setWhereClause(combinePredicate(predicate)); String firstQueryString = CockroachDBVisitor.asString(select); @@ -56,4 +58,9 @@ public CockroachDBExpression combinePredicate(CockroachDBExpression expr) { return new CockroachDBBinaryLogicalOperation(originalPredicate, expr, CockroachDBBinaryLogicalOperator.AND); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java index 34d372d30..466252e56 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java @@ -14,6 +14,8 @@ public class CockroachDBTLPGroupByOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPGroupByOracle(CockroachDBGlobalState state) { super(state); } @@ -24,7 +26,7 @@ public void check() throws SQLException { select.setGroupByExpressions(select.getFetchColumns()); select.setWhereClause(null); String originalQueryString = CockroachDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -42,10 +44,13 @@ public void check() throws SQLException { @Override List generateFetchColumns() { - List columns = new ArrayList<>(); - columns = Randomly.nonEmptySubset(targetTables.getColumns().stream().map(c -> new CockroachDBColumnReference(c)) + return Randomly.nonEmptySubset(targetTables.getColumns().stream().map(c -> new CockroachDBColumnReference(c)) .collect(Collectors.toList())); - return columns; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java index 5aeb8e29b..7a7dd7779 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java @@ -13,6 +13,8 @@ public class CockroachDBTLPHavingOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPHavingOracle(CockroachDBGlobalState state) { super(state); errors.add("GROUP BY term out of range"); @@ -26,18 +28,20 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); String originalQueryString = CockroachDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); select.setHavingClause(predicate); String firstQueryString = CockroachDBVisitor.asString(select); - select.setHavingClause(negatedPredicate); + select.setHavingClause(gen.negatePredicate(predicate)); String secondQueryString = CockroachDBVisitor.asString(select); - select.setHavingClause(isNullPredicate); + select.setHavingClause(gen.isNull(predicate)); String thirdQueryString = CockroachDBVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, @@ -47,8 +51,13 @@ public void check() throws SQLException { } @Override - CockroachDBExpression generatePredicate() { + protected CockroachDBExpression generatePredicate() { return gen.generateHavingClause(); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java deleted file mode 100644 index e65bff7ab..000000000 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java +++ /dev/null @@ -1,119 +0,0 @@ -package sqlancer.cockroachdb.oracle.tlp; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.TestOracle; -import sqlancer.cockroachdb.CockroachDBErrors; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; -import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.cockroachdb.ast.CockroachDBColumnReference; -import sqlancer.cockroachdb.ast.CockroachDBConstant; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBJoin.OuterType; -import sqlancer.cockroachdb.ast.CockroachDBNotOperation; -import sqlancer.cockroachdb.ast.CockroachDBSelect; -import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; -import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; - -public class CockroachDBTLPJoinOracle implements TestOracle { - - final CockroachDBGlobalState state; - final Set errors = new HashSet<>(); - - CockroachDBSchema s; - CockroachDBTables targetTables; - CockroachDBExpressionGenerator gen; - CockroachDBSelect select; - CockroachDBExpression predicate; - CockroachDBExpression negatedPredicate; - CockroachDBExpression isNullPredicate; - - public CockroachDBTLPJoinOracle(CockroachDBGlobalState state) { - errors.add("GROUP BY term out of range"); - CockroachDBErrors.addExpressionErrors(errors); - this.state = state; - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new CockroachDBExpressionGenerator(state).setColumns(targetTables.getColumns()); - select = new CockroachDBSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - if (tables.size() < 2) { - return; - } - List tableList = tables.stream().map(t -> new CockroachDBTableReference(t)) - .collect(Collectors.toList()); - CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBJoin leftJoinTrue = CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, - CockroachDBConstant.createBooleanConstant(true)); - - select.setJoinList(Arrays.asList(leftJoinTrue)); - select.setFromList(tableList); - select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new CockroachDBNotOperation(predicate); - isNullPredicate = new CockroachDBUnaryPostfixOperation(predicate, CockroachDBUnaryPostfixOperator.IS_NULL); - - String originalQueryString1 = CockroachDBVisitor.asString(select); - - CockroachDBJoin leftJoinFalse = CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, - CockroachDBConstant.createBooleanConstant(false)); - select.setJoinList(Arrays.asList(leftJoinFalse)); - String originalQueryString2 = CockroachDBVisitor.asString(select); - String originalQueryString = originalQueryString1 + " UNION ALL " + originalQueryString2 + " UNION ALL " - + originalQueryString2; - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - // boolean allowOrderBy = Randomly.getBoolean(); - // if (allowOrderBy) { - // select.setOrderByExpressions(gen.getOrderingTerms()); - // } - - select.setJoinList( - Arrays.asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, predicate))); - String firstQueryString = CockroachDBVisitor.asString(select); - - select.setJoinList(Arrays - .asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, negatedPredicate))); - String secondQueryString = CockroachDBVisitor.asString(select); - - select.setJoinList( - Arrays.asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, isNullPredicate))); - String thirdQueryString = CockroachDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - - List generateFetchColumns() { - return Arrays.asList(new CockroachDBColumnReference(targetTables.getColumns().get(0))); - } - - CockroachDBExpression generatePredicate() { - return gen.generateExpression(CockroachDBDataType.BOOL.get()); - } - -} diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPOracle.java deleted file mode 100644 index 41d6b9269..000000000 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPOracle.java +++ /dev/null @@ -1,32 +0,0 @@ -package sqlancer.cockroachdb.oracle.tlp; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.TestOracle; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; - -public class CockroachDBTLPOracle implements TestOracle { - - private final TestOracle[] oracles; - private int i; - - public CockroachDBTLPOracle(CockroachDBGlobalState state) { - List oracles = new ArrayList<>(); - oracles.add(new CockroachDBTLPAggregateOracle(state)); - oracles.add(new CockroachDBTLPHavingOracle(state)); - oracles.add(new CockroachDBTLPWhereOracle(state)); - oracles.add(new CockroachDBTLPGroupByOracle(state)); - oracles.add(new CockroachDBTLPExtendedWhereOracle(state)); - oracles.add(new CockroachDBTLPDistinctOracle(state)); - this.oracles = oracles.toArray(new TestOracle[4]); - } - - @Override - public void check() throws SQLException { - oracles[i].check(); - i = (i + 1) % oracles.length; - } - -} diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java deleted file mode 100644 index 8fe0899d0..000000000 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.cockroachdb.oracle.tlp; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBNotOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; - -public class CockroachDBTLPWhereOracle extends CockroachDBTLPBase { - - public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { - super(state); - errors.add("GROUP BY term out of range"); - } - - @Override - public void check() throws SQLException { - super.check(); - String originalQueryString = CockroachDBVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean allowOrderBy = Randomly.getBoolean(); - if (allowOrderBy) { - select.setOrderByExpressions(gen.getOrderingTerms()); - } - CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); - select.setWhereClause(predicate); - String firstQueryString = CockroachDBVisitor.asString(select); - select.setWhereClause(new CockroachDBNotOperation(predicate)); - String secondQueryString = CockroachDBVisitor.asString(select); - select.setWhereClause(new CockroachDBUnaryPostfixOperation(predicate, CockroachDBUnaryPostfixOperator.IS_NULL)); - String thirdQueryString = CockroachDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !allowOrderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } -} diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java new file mode 100644 index 000000000..4478be2b3 --- /dev/null +++ b/src/sqlancer/common/DBMSCommon.java @@ -0,0 +1,69 @@ +package sqlancer.common; + +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class DBMSCommon { + + private static final Pattern SQLANCER_INDEX_PATTERN = Pattern.compile("^i\\d+"); + + private DBMSCommon() { + } + + public static String createTableName(int nr) { + return String.format("t%d", nr); + } + + public static String createColumnName(int nr) { + return String.format("c%d", nr); + } + + public static String createIndexName(int nr) { + return String.format("i%d", nr); + } + + public static boolean matchesIndexName(String indexName) { + Matcher matcher = SQLANCER_INDEX_PATTERN.matcher(indexName); + return matcher.matches(); + } + + public static int getMaxIndexInDoubleArray(double... doubleArray) { + int maxIndex = 0; + double maxValue = 0.0; + for (int j = 0; j < doubleArray.length; j++) { + double curReward = doubleArray[j]; + if (curReward > maxValue) { + maxIndex = j; + maxValue = curReward; + } + } + return maxIndex; + } + + public static boolean areQueryPlanSequencesSimilar(List list1, List list2) { + return editDistance(list1, list2) <= 1; + } + + public static int editDistance(List list1, List list2) { + int[][] dp = new int[list1.size() + 1][list2.size() + 1]; + for (int i = 0; i <= list1.size(); i++) { + for (int j = 0; j <= list2.size(); j++) { + if (i == 0) { + dp[i][j] = j; + } else if (j == 0) { + dp[i][j] = i; + } else { + dp[i][j] = Math.min(dp[i - 1][j - 1] + costOfSubstitution(list1.get(i - 1), list2.get(j - 1)), + Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1)); + } + } + } + return dp[list1.size()][list2.size()]; + } + + private static int costOfSubstitution(String string, String string2) { + return string.equals(string2) ? 0 : 1; + } + +} diff --git a/src/sqlancer/ast/BinaryNode.java b/src/sqlancer/common/ast/BinaryNode.java similarity index 72% rename from src/sqlancer/ast/BinaryNode.java rename to src/sqlancer/common/ast/BinaryNode.java index 5be42b654..90a56a078 100644 --- a/src/sqlancer/ast/BinaryNode.java +++ b/src/sqlancer/common/ast/BinaryNode.java @@ -1,13 +1,13 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.visitor.BinaryOperation; +import sqlancer.common.visitor.BinaryOperation; public abstract class BinaryNode implements BinaryOperation { private final T left; private final T right; - public BinaryNode(T left, T right) { + protected BinaryNode(T left, T right) { this.left = left; this.right = right; } diff --git a/src/sqlancer/ast/BinaryOperatorNode.java b/src/sqlancer/common/ast/BinaryOperatorNode.java similarity index 74% rename from src/sqlancer/ast/BinaryOperatorNode.java rename to src/sqlancer/common/ast/BinaryOperatorNode.java index 002cab790..586315059 100644 --- a/src/sqlancer/ast/BinaryOperatorNode.java +++ b/src/sqlancer/common/ast/BinaryOperatorNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public abstract class BinaryOperatorNode extends BinaryNode { @@ -10,7 +10,7 @@ public interface Operator { String getTextRepresentation(); } - public BinaryOperatorNode(T left, T right, O op) { + protected BinaryOperatorNode(T left, T right, O op) { super(left, right); this.op = op; } diff --git a/src/sqlancer/ast/FunctionNode.java b/src/sqlancer/common/ast/FunctionNode.java similarity index 78% rename from src/sqlancer/ast/FunctionNode.java rename to src/sqlancer/common/ast/FunctionNode.java index 6527a3725..ad795b998 100644 --- a/src/sqlancer/ast/FunctionNode.java +++ b/src/sqlancer/common/ast/FunctionNode.java @@ -1,4 +1,4 @@ -package sqlancer.ast; +package sqlancer.common.ast; import java.util.List; @@ -7,7 +7,7 @@ public abstract class FunctionNode { protected F function; protected List args; - public FunctionNode(F function, List args) { + protected FunctionNode(F function, List args) { this.function = function; this.args = args; } diff --git a/src/sqlancer/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java similarity index 80% rename from src/sqlancer/ast/SelectBase.java rename to src/sqlancer/common/ast/SelectBase.java index 627bcd119..e79a1a87d 100644 --- a/src/sqlancer/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -1,4 +1,4 @@ -package sqlancer.ast; +package sqlancer.common.ast; import java.util.Collections; import java.util.List; @@ -36,6 +36,10 @@ public void setFromList(List fromList) { this.fromList = fromList; } + public void setFromTables(List tables) { + setFromList(tables); + } + public List getFromList() { if (fromList == null) { throw new IllegalStateException(); @@ -50,19 +54,23 @@ public void setGroupByExpressions(List groupByExpressions) { this.groupByExpressions = groupByExpressions; } + public void clearGroupByExpressions() { + this.groupByExpressions = Collections.emptyList(); + } + public List getGroupByExpressions() { assert groupByExpressions != null; return groupByExpressions; } - public void setOrderByExpressions(List orderByExpressions) { + public void setOrderByClauses(List orderByExpressions) { if (orderByExpressions == null) { throw new IllegalArgumentException(); } this.orderByExpressions = orderByExpressions; } - public List getOrderByExpressions() { + public List getOrderByClauses() { assert orderByExpressions != null; return orderByExpressions; } @@ -83,6 +91,10 @@ public T getHavingClause() { return havingClause; } + public void clearHavingClause() { + this.havingClause = null; + } + public void setLimitClause(T limitClause) { this.limitClause = limitClause; } @@ -107,4 +119,11 @@ public void setJoinList(List joinList) { this.joinList = joinList; } + public List getGroupByClause() { + return getGroupByExpressions(); + } + + public void setGroupByClause(List groupByExpressions) { + setGroupByExpressions(groupByExpressions); + } } diff --git a/src/sqlancer/ast/TernaryNode.java b/src/sqlancer/common/ast/TernaryNode.java similarity index 76% rename from src/sqlancer/ast/TernaryNode.java rename to src/sqlancer/common/ast/TernaryNode.java index fa96833f8..1a6605317 100644 --- a/src/sqlancer/ast/TernaryNode.java +++ b/src/sqlancer/common/ast/TernaryNode.java @@ -1,6 +1,6 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.visitor.BinaryOperation; +import sqlancer.common.visitor.BinaryOperation; public abstract class TernaryNode implements BinaryOperation { @@ -8,7 +8,7 @@ public abstract class TernaryNode implements BinaryOperation { private final T middle; private final T right; - public TernaryNode(T left, T middle, T right) { + protected TernaryNode(T left, T middle, T right) { this.left = left; this.middle = middle; this.right = right; diff --git a/src/sqlancer/ast/UnaryNode.java b/src/sqlancer/common/ast/UnaryNode.java similarity index 65% rename from src/sqlancer/ast/UnaryNode.java rename to src/sqlancer/common/ast/UnaryNode.java index 6b1e5a485..dea4220b9 100644 --- a/src/sqlancer/ast/UnaryNode.java +++ b/src/sqlancer/common/ast/UnaryNode.java @@ -1,12 +1,12 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public abstract class UnaryNode implements UnaryOperation { protected final T expr; - public UnaryNode(T expr) { + protected UnaryNode(T expr) { this.expr = expr; } diff --git a/src/sqlancer/ast/UnaryOperatorNode.java b/src/sqlancer/common/ast/UnaryOperatorNode.java similarity index 67% rename from src/sqlancer/ast/UnaryOperatorNode.java rename to src/sqlancer/common/ast/UnaryOperatorNode.java index e54231929..71556496c 100644 --- a/src/sqlancer/ast/UnaryOperatorNode.java +++ b/src/sqlancer/common/ast/UnaryOperatorNode.java @@ -1,12 +1,12 @@ -package sqlancer.ast; +package sqlancer.common.ast; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; public abstract class UnaryOperatorNode extends UnaryNode { protected final O op; - public UnaryOperatorNode(T expr, O op) { + protected UnaryOperatorNode(T expr, O op) { super(expr); this.op = op; } diff --git a/src/sqlancer/ast/newast/ColumnReferenceNode.java b/src/sqlancer/common/ast/newast/ColumnReferenceNode.java similarity index 67% rename from src/sqlancer/ast/newast/ColumnReferenceNode.java rename to src/sqlancer/common/ast/newast/ColumnReferenceNode.java index cfb506431..2d8688d8f 100644 --- a/src/sqlancer/ast/newast/ColumnReferenceNode.java +++ b/src/sqlancer/common/ast/newast/ColumnReferenceNode.java @@ -1,8 +1,8 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; -import sqlancer.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTableColumn; -public class ColumnReferenceNode> implements Node { +public class ColumnReferenceNode> { private final C c; diff --git a/src/sqlancer/common/ast/newast/Constant.java b/src/sqlancer/common/ast/newast/Constant.java new file mode 100644 index 000000000..a6a6dfd49 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Constant.java @@ -0,0 +1,6 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTableColumn; + +public interface Constant> extends Expression { +} diff --git a/src/sqlancer/common/ast/newast/Expression.java b/src/sqlancer/common/ast/newast/Expression.java new file mode 100644 index 000000000..925cb82c5 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Expression.java @@ -0,0 +1,6 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTableColumn; + +public interface Expression> { +} diff --git a/src/sqlancer/common/ast/newast/Join.java b/src/sqlancer/common/ast/newast/Join.java new file mode 100644 index 000000000..94d26c139 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Join.java @@ -0,0 +1,10 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; + +public interface Join, T extends AbstractTable, C extends AbstractTableColumn> + extends Expression { + + void setOnClause(E onClause); +} diff --git a/src/sqlancer/common/ast/newast/NewAliasNode.java b/src/sqlancer/common/ast/newast/NewAliasNode.java new file mode 100644 index 000000000..260900712 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewAliasNode.java @@ -0,0 +1,21 @@ +package sqlancer.common.ast.newast; + +public class NewAliasNode { + + private final E expr; + private final String alias; + + public NewAliasNode(E expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + public E getExpr() { + return expr; + } + + public String getAlias() { + return alias; + } + +} diff --git a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java new file mode 100644 index 000000000..4bb9ce17a --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java @@ -0,0 +1,33 @@ +package sqlancer.common.ast.newast; + +public class NewBetweenOperatorNode { + + protected T left; + protected T middle; + protected T right; + protected boolean isTrue; + + public NewBetweenOperatorNode(T left, T middle, T right, boolean isTrue) { + this.left = left; + this.middle = middle; + this.right = right; + this.isTrue = isTrue; + } + + public T getLeft() { + return left; + } + + public T getMiddle() { + return middle; + } + + public T getRight() { + return right; + } + + public boolean isTrue() { + return isTrue; + } + +} diff --git a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java new file mode 100644 index 000000000..b2fcc5ef0 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java @@ -0,0 +1,29 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +public class NewBinaryOperatorNode { + + protected final Operator op; + protected final T left; + protected final T right; + + public NewBinaryOperatorNode(T left, T right, Operator op) { + this.left = left; + this.right = right; + this.op = op; + } + + public String getOperatorRepresentation() { + return op.getTextRepresentation(); + } + + public T getLeft() { + return left; + } + + public T getRight() { + return right; + } + +} diff --git a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java new file mode 100644 index 000000000..440078566 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java @@ -0,0 +1,38 @@ +package sqlancer.common.ast.newast; + +import java.util.List; + +public class NewCaseOperatorNode { + + private final List conditions; + private final List expressions; + private final T elseExpr; + private final T switchCondition; + + public NewCaseOperatorNode(T switchCondition, List conditions, List expressions, T elseExpr) { + this.switchCondition = switchCondition; + this.conditions = conditions; + this.expressions = expressions; + this.elseExpr = elseExpr; + if (conditions.size() != expressions.size()) { + throw new IllegalArgumentException(); + } + } + + public T getSwitchCondition() { + return switchCondition; + } + + public List getConditions() { + return conditions; + } + + public List getExpressions() { + return expressions; + } + + public T getElseExpr() { + return elseExpr; + } + +} diff --git a/src/sqlancer/common/ast/newast/NewFunctionNode.java b/src/sqlancer/common/ast/newast/NewFunctionNode.java new file mode 100644 index 000000000..cdd91bd61 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewFunctionNode.java @@ -0,0 +1,23 @@ +package sqlancer.common.ast.newast; + +import java.util.List; + +public class NewFunctionNode { + + protected List args; + protected F func; + + public NewFunctionNode(List args, F func) { + this.args = args; + this.func = func; + } + + public List getArgs() { + return args; + } + + public F getFunc() { + return func; + } + +} diff --git a/src/sqlancer/common/ast/newast/NewInOperatorNode.java b/src/sqlancer/common/ast/newast/NewInOperatorNode.java new file mode 100644 index 000000000..94a3a0886 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewInOperatorNode.java @@ -0,0 +1,29 @@ +package sqlancer.common.ast.newast; + +import java.util.List; + +public class NewInOperatorNode { + + private final T left; + private final List right; + private final boolean isNegated; + + public NewInOperatorNode(T left, List right, boolean isNegated) { + this.left = left; + this.right = right; + this.isNegated = isNegated; + } + + public T getLeft() { + return left; + } + + public List getRight() { + return right; + } + + public boolean isNegated() { + return isNegated; + } + +} diff --git a/src/sqlancer/ast/newast/NewOrderingTerm.java b/src/sqlancer/common/ast/newast/NewOrderingTerm.java similarity index 65% rename from src/sqlancer/ast/newast/NewOrderingTerm.java rename to src/sqlancer/common/ast/newast/NewOrderingTerm.java index 6328070ae..2efed05bc 100644 --- a/src/sqlancer/ast/newast/NewOrderingTerm.java +++ b/src/sqlancer/common/ast/newast/NewOrderingTerm.java @@ -1,10 +1,10 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; import sqlancer.Randomly; -public class NewOrderingTerm implements Node { +public class NewOrderingTerm { - private final Node expr; + private final T expr; private final Ordering ordering; public enum Ordering { @@ -15,12 +15,12 @@ public static Ordering getRandom() { } } - public NewOrderingTerm(Node expr, Ordering ordering) { + public NewOrderingTerm(T expr, Ordering ordering) { this.expr = expr; this.ordering = ordering; } - public Node getExpr() { + public T getExpr() { return expr; } diff --git a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java new file mode 100644 index 000000000..716dbf8e3 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java @@ -0,0 +1,20 @@ +package sqlancer.common.ast.newast; + +public class NewPostfixTextNode { + + private final T expr; + private final String text; + + public NewPostfixTextNode(T expr, String text) { + this.expr = expr; + this.text = text; + } + + public T getExpr() { + return expr; + } + + public String getText() { + return text; + } +} diff --git a/src/sqlancer/ast/newast/NewTernaryNode.java b/src/sqlancer/common/ast/newast/NewTernaryNode.java similarity index 53% rename from src/sqlancer/ast/newast/NewTernaryNode.java rename to src/sqlancer/common/ast/newast/NewTernaryNode.java index 1ec9a6c35..2ca7b1f30 100644 --- a/src/sqlancer/ast/newast/NewTernaryNode.java +++ b/src/sqlancer/common/ast/newast/NewTernaryNode.java @@ -1,14 +1,14 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; -public class NewTernaryNode implements Node { +public class NewTernaryNode { - protected final Node left; - protected final Node middle; - protected final Node right; + protected final T left; + protected final T middle; + protected final T right; private final String leftStr; private final String rightStr; - public NewTernaryNode(Node left, Node middle, Node right, String leftStr, String rightStr) { + public NewTernaryNode(T left, T middle, T right, String leftStr, String rightStr) { this.left = left; this.middle = middle; this.right = right; @@ -16,15 +16,15 @@ public NewTernaryNode(Node left, Node middle, Node right, String leftSt this.rightStr = rightStr; } - public Node getLeft() { + public T getLeft() { return left; } - public Node getMiddle() { + public T getMiddle() { return middle; } - public Node getRight() { + public T getRight() { return right; } diff --git a/src/sqlancer/ast/newast/NewToStringVisitor.java b/src/sqlancer/common/ast/newast/NewToStringVisitor.java similarity index 96% rename from src/sqlancer/ast/newast/NewToStringVisitor.java rename to src/sqlancer/common/ast/newast/NewToStringVisitor.java index 6c275684a..82b6bace2 100644 --- a/src/sqlancer/ast/newast/NewToStringVisitor.java +++ b/src/sqlancer/common/ast/newast/NewToStringVisitor.java @@ -1,4 +1,4 @@ -package sqlancer.ast.newast; +package sqlancer.common.ast.newast; import java.util.List; @@ -7,7 +7,7 @@ public abstract class NewToStringVisitor { protected final StringBuilder sb = new StringBuilder(); @SuppressWarnings("unchecked") - public void visit(Node expr) { + public void visit(E expr) { assert expr != null; if (expr instanceof ColumnReferenceNode) { sb.append(((ColumnReferenceNode) expr).getColumn().getFullQualifiedName()); @@ -40,7 +40,7 @@ public void visit(Node expr) { } } - public void visit(List> expressions) { + public void visit(List expressions) { for (int i = 0; i < expressions.size(); i++) { if (i != 0) { sb.append(", "); @@ -165,6 +165,6 @@ public String get() { return sb.toString(); } - public abstract void visitSpecific(Node expr); + public abstract void visitSpecific(E expr); } diff --git a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java new file mode 100644 index 000000000..b3ccd4cd3 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java @@ -0,0 +1,22 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +public class NewUnaryPostfixOperatorNode { + + protected final Operator op; + private final T expr; + + public NewUnaryPostfixOperatorNode(T expr, Operator op) { + this.expr = expr; + this.op = op; + } + + public String getOperatorRepresentation() { + return op.getTextRepresentation(); + } + + public T getExpr() { + return expr; + } +} diff --git a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java new file mode 100644 index 000000000..8668aec35 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java @@ -0,0 +1,23 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +public class NewUnaryPrefixOperatorNode { + + protected final Operator op; + private final T expr; + + public NewUnaryPrefixOperatorNode(T expr, Operator op) { + this.expr = expr; + this.op = op; + } + + public String getOperatorRepresentation() { + return op.getTextRepresentation(); + } + + public T getExpr() { + return expr; + } + +} diff --git a/src/sqlancer/common/ast/newast/Select.java b/src/sqlancer/common/ast/newast/Select.java new file mode 100644 index 000000000..53520648d --- /dev/null +++ b/src/sqlancer/common/ast/newast/Select.java @@ -0,0 +1,48 @@ +package sqlancer.common.ast.newast; + +import java.util.List; + +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; + +public interface Select, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> + extends Expression { + + List getFromList(); + + void setFromList(List fromList); + + Expression getWhereClause(); + + void setWhereClause(E whereClause); + + void setGroupByClause(List groupByClause); + + List getGroupByClause(); + + void setLimitClause(E limitClause); + + Expression getLimitClause(); + + List getOrderByClauses(); + + void setOrderByClauses(List orderBy); + + void setOffsetClause(E offsetClause); + + Expression getOffsetClause(); + + void setFetchColumns(List fetchColumns); + + List getFetchColumns(); + + void setJoinClauses(List joinStatements); + + List getJoinClauses(); + + void setHavingClause(E havingClause); + + Expression getHavingClause(); + + String asString(); +} diff --git a/src/sqlancer/common/ast/newast/TableReferenceNode.java b/src/sqlancer/common/ast/newast/TableReferenceNode.java new file mode 100644 index 000000000..0719a0ed6 --- /dev/null +++ b/src/sqlancer/common/ast/newast/TableReferenceNode.java @@ -0,0 +1,17 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTable; + +public class TableReferenceNode> { + + private final T t; + + public TableReferenceNode(T table) { + this.t = table; + } + + public T getTable() { + return t; + } + +} diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java new file mode 100644 index 000000000..f3b5a1955 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -0,0 +1,58 @@ +package sqlancer.common.gen; + +public abstract class AbstractDeleteGenerator extends AbstractGenerator { + + protected AbstractDeleteGenerator() { + } + + /** + * Appends {@code DELETE FROM }. + * + * @param tableName + * the name of the table to delete from. + */ + protected void appendDeleteFromTable(String tableName) { + appendDeleteFromTable(tableName, false); + } + + /** + * Appends {@code DELETE FROM [ONLY ]}. + * + * @param tableName + * the name of the table to delete from. + * @param only + * whether to emit the {@code ONLY} keyword (used by some databases to restrict deletion to the named + * table rather than its inheritance descendants). + */ + protected void appendDeleteFromTable(String tableName, boolean only) { + sb.append("DELETE FROM "); + if (only) { + sb.append("ONLY "); + } + sb.append(tableName); + } + + /** + * Appends {@code LIMIT } (with a leading space). + * + * @param value + * the LIMIT value, e.g. an integer literal or already-rendered expression. Converted via + * {@link StringBuilder#append(Object)}. + */ + protected void appendLimitClause(Object value) { + sb.append(" LIMIT "); + sb.append(value); + } + + /** + * Appends {@code RETURNING } (with a leading space). + * + * @param expression + * the rendered RETURNING expression. + */ + protected void appendReturningClause(String expression) { + sb.append(" RETURNING "); + sb.append(expression); + } + +} diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java new file mode 100644 index 000000000..dbdf100d5 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.common.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public abstract class AbstractGenerator { + + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final StringBuilder sb = new StringBuilder(); + protected boolean canAffectSchema; + protected boolean canonicalizeString = true; + + public SQLQueryAdapter getStatement() { + buildStatement(); + return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema, canonicalizeString); + } + + public abstract void buildStatement(); + + /** + * Appends {@code WHERE } (with a leading space). Subclasses are responsible for deciding whether to + * include the WHERE clause, typically based on a randomized boolean. Used by DELETE, UPDATE, partial-INDEX, and + * INSERT...ON CONFLICT generators. + * + * @param condition + * the rendered WHERE condition. + */ + protected void appendWhereClause(String condition) { + sb.append(" WHERE "); + sb.append(condition); + } + +} diff --git a/src/sqlancer/common/gen/AbstractIndexGenerator.java b/src/sqlancer/common/gen/AbstractIndexGenerator.java new file mode 100644 index 000000000..bfa62a36c --- /dev/null +++ b/src/sqlancer/common/gen/AbstractIndexGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractIndexGenerator> extends AbstractGenerator { + + protected void appendCreateIndex(boolean unique) { + sb.append("CREATE "); + if (unique) { + sb.append("UNIQUE "); + } + sb.append("INDEX "); + } + + protected void appendIndexColumnList(List columns, boolean allowOrdering) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + if (allowOrdering && Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java new file mode 100644 index 000000000..1a1b36b67 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -0,0 +1,43 @@ +package sqlancer.common.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractInsertGenerator> extends AbstractGenerator { + + protected void appendColumnList(List columns) { + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + } + + protected void buildInsertInto(String tableName, List columns) { + sb.append("INSERT INTO "); + sb.append(tableName); + appendColumnList(columns); + sb.append(" VALUES "); + insertColumns(columns); + } + + protected void insertColumns(List columns) { + for (int nrRows = 0; nrRows < Randomly.smallNumber() + 1; nrRows++) { + if (nrRows != 0) { + sb.append(", "); + } + sb.append("("); + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + insertValue(columns.get(nrColumn)); + } + sb.append(")"); + } + } + + protected abstract void insertValue(C column); + +} diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java new file mode 100644 index 000000000..c8d1d9697 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -0,0 +1,77 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractTableGenerator> extends AbstractGenerator { + + /** + * Appends {@code CREATE TABLE }. + * + * @param tableName + * the name of the table to create. + */ + protected void appendCreateTable(String tableName) { + appendCreateTable(tableName, false); + } + + /** + * Appends {@code CREATE TABLE [IF NOT EXISTS ]}. + * + * @param tableName + * the name of the table to create. + * @param ifNotExists + * whether to emit the {@code IF NOT EXISTS} clause. + */ + protected void appendCreateTable(String tableName, boolean ifNotExists) { + sb.append("CREATE TABLE "); + if (ifNotExists) { + sb.append("IF NOT EXISTS "); + } + sb.append(tableName); + } + + /** + * Appends a parenthesized, comma-separated column definition list, e.g. {@code (c0 INT, c1 TEXT)}. Delegates each + * column's rendering to {@link #appendColumnDefinition(AbstractTableColumn)}. + * + * @param columns + * the columns to render. + */ + protected void appendColumnDefinitions(List columns) { + sb.append("("); + appendColumnDefinitionList(columns); + sb.append(")"); + } + + /** + * Appends a comma-separated column definition list without enclosing parentheses, e.g. {@code c0 INT, c1 TEXT}. + * Useful when subclasses also emit table-level constraints (e.g. {@code PRIMARY KEY (...)}) inside the same parens. + * + * @param columns + * the columns to render. + */ + protected void appendColumnDefinitionList(List columns) { + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumnDefinition(columns.get(i)); + } + } + + /** + * Appends a single column's definition. Default output is {@code }, e.g. {@code c0 INT}. Override to + * add constraints such as {@code NOT NULL}, {@code DEFAULT ...}, or {@code CHECK (...)}. + * + * @param column + * the column whose definition to render. + */ + protected void appendColumnDefinition(C column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + } + +} diff --git a/src/sqlancer/common/gen/AbstractUpdateGenerator.java b/src/sqlancer/common/gen/AbstractUpdateGenerator.java new file mode 100644 index 000000000..52b716bff --- /dev/null +++ b/src/sqlancer/common/gen/AbstractUpdateGenerator.java @@ -0,0 +1,22 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractUpdateGenerator> extends AbstractGenerator { + + protected void updateColumns(List columns) { + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + sb.append(columns.get(nrColumn).getName()); + sb.append("="); + updateValue(columns.get(nrColumn)); + } + } + + protected abstract void updateValue(C column); + +} diff --git a/src/sqlancer/common/gen/CERTGenerator.java b/src/sqlancer/common/gen/CERTGenerator.java new file mode 100644 index 000000000..b272ba4b2 --- /dev/null +++ b/src/sqlancer/common/gen/CERTGenerator.java @@ -0,0 +1,29 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface CERTGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + CERTGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + String generateExplainQuery(S select); + + boolean mutate(S select); +} diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java new file mode 100644 index 000000000..9dbc9f0d5 --- /dev/null +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -0,0 +1,340 @@ +package sqlancer.common.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.oracle.EETTransformer; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETDMLOracle}, the DML counterpart of {@link EETGenerator}. + * It supplies methods which generate the transformable expressions, create the DBMS-specific {@link EETTransformer} + * that rewrites them, and produce the SQL of the statements the oracle uses to observe the database state a statement + * produces (an approach drawn from the DQE oracle). + * + *

+ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which + * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run + * (each in a rolled-back transaction), so both executions observe the same identifiers. The resulting state is compared + * as a full post-image (each surviving row's identifier and content column values), which covers any of the three DML + * statements (DELETE, UPDATE, INSERT). + * + *

+ * Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ +public interface EETDMLGenerator, T extends AbstractTable, C extends AbstractTableColumn> { + + /** Name of the auxiliary column that uniquely identifies each row. */ + String ROW_ID_COLUMN = "rowid"; + + /** + * Restricts this generator to the given tables (a single table, for the DML statement under test) and their + * columns. + * + * @param tables + * the tables (and, implicitly, columns) the generated statement operates on + * + * @return this generator + */ + EETDMLGenerator setTablesAndColumns(AbstractTables tables); + + /** + * Generates a fresh random boolean expression over the current tables' columns, used as the DML statement's WHERE + * predicate. + * + * @return a fresh random boolean expression + */ + E generateBooleanExpression(); + + /** + * Generates a fresh set of {@code column = value} assignments over the current tables' columns, used as an UPDATE + * statement's SET clause. The columns are a random non-empty subset and each value is a fresh random expression; + * both the columns and their assigned expressions are transformed by the oracle. + * + * @return the assignments, as {@code (column, value expression)} pairs (at least one) + */ + List> generateSetAssignments(); + + /** + * Generates a fresh value expression for each content column of the current table, used as an INSERT statement's + * inserted values. The returned expressions are positionally aligned with {@link AbstractTable#getColumns()}, and + * each is transformed by the oracle. + * + * @return one fresh random value expression per content column, in {@link AbstractTable#getColumns()} order + */ + List generateInsertValues(); + + /** + * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's + * expressions into semantically equivalent ones. + * + * @return a DBMS-specific {@link EETTransformer} + */ + EETTransformer createTransformer(); + + // --- DBMS-specific primitives --- + + /** + * Renders an expression to its DBMS-specific SQL string. + * + * @param expr + * the expression to render + * + * @return the SQL text of {@code expr} + */ + String asString(E expr); + + /** + * SQL that assigns every existing row of {@code table} a distinct, stable identifier in the {@link #ROW_ID_COLUMN} + * column. For example, a 36-character UUID string. + * + * @param table + * the table whose rows are stamped + * + * @return the SQL statement + */ + String stampRowIdsStatement(T table); + + /** + * The SQL type of the auxiliary {@link #ROW_ID_COLUMN} column. It must be able to hold the identifiers that + * {@link #stampRowIdsStatement} produces, so it belongs with that statement as the other half of the row-id + * representation. For example, {@code VARCHAR(36)} would fit a 36-character UUID string. + * + * @return the column type + */ + String rowIdColumnType(); + + /** + * A SQL expression, evaluated once per source row of an {@code INSERT ... SELECT}, that derives the inserted row's + * {@link #ROW_ID_COLUMN} value from the source row's identifier. It must be deterministic (so both the original and + * transformed statements assign the same identifiers), unique per source row, and distinct from every existing + * identifier (so an inserted row never collides with the source row it was derived from in the post-image). DBMS- + * specific because it names a suitable derivation function (e.g. a hash of the source identifier). + * + * @return the SQL expression deriving an inserted row's identifier from the source row's {@link #ROW_ID_COLUMN} + */ + String insertedRowIdExpression(); + + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- + + /** + * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}, typed as {@link #rowIdColumnType}. + * + * @param table + * the table to add the column to + * + * @return the SQL statement + */ + default String addRowIdColumnStatement(T table) { + return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " " + rowIdColumnType(); + } + + /** + * SQL that drops the auxiliary {@link #ROW_ID_COLUMN} column from {@code table}. + * + * @param table + * the table to drop the column from + * + * @return the SQL statement + */ + default String dropRowIdColumnStatement(T table) { + return "ALTER TABLE " + table.getName() + " DROP COLUMN " + ROW_ID_COLUMN; + } + + /** + * SQL that reads back the full post-image of {@code table}: the {@link #ROW_ID_COLUMN} identifier and every content + * column of every surviving row, ordered by the (unique) identifier so the two statements' snapshots align + * row-for-row. + * + *

+ * This single value-level snapshot is the comparison surface for all DML statements: a DELETE removes rows from it, + * an UPDATE changes column values in it, an INSERT adds rows to it. Row identity alone (which the identifier + * already captures) would suffice for DELETE, but not for UPDATE, where the two runs could touch the same rows yet + * write different values. + * + * @param table + * the table to snapshot + * + * @return the SQL statement; its result columns are those of {@link #postImageColumns}, in that order + */ + default String selectPostImageStatement(T table) { + return "SELECT " + String.join(", ", postImageColumns(table)) + " FROM " + table.getName() + " ORDER BY " + + ROW_ID_COLUMN; + } + + /** + * The columns a post-image row consists of, in the order {@link #selectPostImageStatement} returns them: the + * {@link #ROW_ID_COLUMN} identifier followed by {@code table}'s content columns. This is the sole definition of the + * post-image layout, so a consumer can find the identifier's position by looking up {@link #ROW_ID_COLUMN} here + * rather than assuming one. + * + * @param table + * the table being snapshot + * + * @return the post-image column names, in order + */ + default List postImageColumns(T table) { + List columns = new ArrayList<>(); + columns.add(ROW_ID_COLUMN); + for (C column : table.getColumns()) { + columns.add(column.getName()); + } + return columns; + } + + /** + * SQL that deletes the rows of {@code table} matching {@code predicate}, optionally limited to the first + * {@code limit} rows (see {@link #orderByLimitClause}). + * + * @param table + * the table to delete from + * @param predicate + * the WHERE predicate; rendered via {@link #asString} + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty); only used when {@code limit} is + * non-null + * @param limit + * the maximum number of rows to delete, or {@code null} for no limit + * + * @return the SQL statement + */ + default String deleteStatement(T table, E predicate, List orderByColumns, Integer limit) { + return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate) + + orderByLimitClause(orderByColumns, limit); + } + + /** + * SQL that updates the rows of {@code table} matching {@code predicate}, setting each column in {@code assignments} + * to its assigned value expression, optionally limited to the first {@code limit} rows (see + * {@link #orderByLimitClause}). + * + * @param table + * the table to update + * @param assignments + * the {@code (column, value expression)} pairs to assign; each value is rendered via {@link #asString} + * @param predicate + * the WHERE predicate; rendered via {@link #asString} + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty); only used when {@code limit} is + * non-null + * @param limit + * the maximum number of rows to update, or {@code null} for no limit + * + * @return the SQL statement + */ + default String updateStatement(T table, List> assignments, E predicate, List orderByColumns, + Integer limit) { + List setClauses = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + setClauses.add(assignment.getKey().getName() + " = " + asString(assignment.getValue())); + } + return "UPDATE " + table.getName() + " SET " + String.join(", ", setClauses) + " WHERE " + asString(predicate) + + orderByLimitClause(orderByColumns, limit); + } + + /** + * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), + * setting each content column to its corresponding value in {@code values}, optionally limited to the first + * {@code limit} source rows (see {@link #orderByLimitClause}). + * + *

+ * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because it reuses the source-row + * model already shared by {@link #deleteStatement} and {@link #updateStatement}, and because it offers two kinds of + * transformable expression in one statement (the inserted values and the WHERE predicate) rather than the values + * alone. Each inserted row's {@link #ROW_ID_COLUMN} is derived from its source row via + * {@link #insertedRowIdExpression()}, giving it a deterministic identifier that is unique and distinct from every + * existing one, so the two statements' post-images align (and inserted rows never collide with their source rows). + * + * @param table + * the table to insert into + * @param values + * one value expression per content column, positionally aligned with {@link AbstractTable#getColumns()}; + * each is rendered via {@link #asString} + * @param predicate + * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; + * rendered via {@link #asString} + * @param orderByColumns + * the columns to order the source rows by before the row-id tiebreaker (may be empty); only used when + * {@code limit} is non-null + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit + * + * @return the SQL statement + */ + default String insertStatement(T table, List values, E predicate, List orderByColumns, Integer limit) { + List columnNames = new ArrayList<>(); + columnNames.add(ROW_ID_COLUMN); + List selectItems = new ArrayList<>(); + selectItems.add(insertedRowIdExpression()); + List columns = table.getColumns(); + for (int i = 0; i < columns.size(); i++) { + columnNames.add(columns.get(i).getName()); + selectItems.add(asString(values.get(i))); + } + String statement = "INSERT INTO " + table.getName() + " (" + String.join(", ", columnNames) + ") SELECT " + + String.join(", ", selectItems) + " FROM " + table.getName(); + if (predicate != null) { + statement += " WHERE " + asString(predicate); + } + return statement + orderByLimitClause(orderByColumns, limit); + } + + /** + * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement}, + * {@link #updateStatement} and {@link #insertStatement}, or the empty string when {@code limit} is null. + * + *

+ * The rows are ordered by {@code orderByColumns} followed by {@link #ROW_ID_COLUMN} as a tiebreaker. Because the + * identifiers are unique, this is always a total order (even when the ordering columns tie), so the "first + * {@code limit}" rows are identical for the original and transformed statements. Varying the ordering columns + * exercises more access paths than the row id alone would. The caller must pass the same {@code orderByColumns} and + * {@code limit} to both statements; neither is transformed. + * + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty) + * @param limit + * the maximum number of rows, or {@code null} for no limit (yielding an empty clause) + * + * @return the {@code ORDER BY ... LIMIT n} clause, or the empty string when {@code limit} is null + */ + default String orderByLimitClause(List orderByColumns, Integer limit) { + if (limit == null) { + return ""; + } + List orderBy = new ArrayList<>(); + for (C column : orderByColumns) { + orderBy.add(column.getName()); + } + orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above + return " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; + } + + /** + * SQL that starts a transaction, so a statement's effect can be observed and then undone. + * + * @return the SQL statement + */ + default String beginTransactionStatement() { + return "BEGIN"; + } + + /** + * SQL that rolls the current transaction back, undoing the statement's effect. + * + * @return the SQL statement + */ + default String rollbackTransactionStatement() { + return "ROLLBACK"; + } +} diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java new file mode 100644 index 000000000..a87a635ec --- /dev/null +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -0,0 +1,51 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.oracle.EETTransformer; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like + * the other oracle generators), an EET generator creates a DBMS-specific {@link EETTransformer} that the oracle uses to + * rewrite expressions into semantically equivalent ones. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ +public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + EETGenerator setTablesAndColumns(AbstractTables tables); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + E generateBooleanExpression(); + + /** + * Creates a DBMS-specific {@link EETTransformer} backed by this generator. Called once by + * {@link sqlancer.common.oracle.EETOracle} during construction; the oracle owns the returned transformer for the + * lifetime of the test run. + * + * @return a DBMS-specific {@link EETTransformer} backed by this generator + */ + EETTransformer createTransformer(); +} diff --git a/src/sqlancer/common/gen/ExpressionGenerator.java b/src/sqlancer/common/gen/ExpressionGenerator.java new file mode 100644 index 000000000..4d5cff1e2 --- /dev/null +++ b/src/sqlancer/common/gen/ExpressionGenerator.java @@ -0,0 +1,32 @@ +package sqlancer.common.gen; + +public interface ExpressionGenerator { + + /** + * Generates a boolean predicate. + * + * @return an expression that can be used in a boolean context. + */ + E generatePredicate(); + + /** + * Negates a predicate (i.e., uses a NOT operator). + * + * @param predicate + * the boolean predicate. + * + * @return the negated predicate. + */ + E negatePredicate(E predicate); + + /** + * Checks if an expression evaluates to NULL (i.e., implements the IS NULL operator). + * + * @param expr + * the expression + * + * @return an expression that checks whether the expression evaluates to NULL. + */ + E isNull(E expr); + +} diff --git a/src/sqlancer/common/gen/NoRECGenerator.java b/src/sqlancer/common/gen/NoRECGenerator.java new file mode 100644 index 000000000..185e8286b --- /dev/null +++ b/src/sqlancer/common/gen/NoRECGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface NoRECGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + NoRECGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + /** + * Generates a query string that is likely to be optimized by the DBMS. + * + * @param select + * the base select expression used to generate the query + * @param whereCondition + * a condition where records will be checked with + * @param shouldUseAggregate + * whether to aggregate the record counts (`true`) or display records as is (`false`) + * + * @return a query string to be executed + */ + String generateOptimizedQueryString(S select, E whereCondition, boolean shouldUseAggregate); + + /** + * Generates a query string that is unlikely to be optimized by the DBMS. + * + * @param select + * the base select expression used to generate the query + * @param whereCondition + * the condition each record will be checked with + * + * @return a query string to be executed + */ + String generateUnoptimizedQueryString(S select, E whereCondition); +} diff --git a/src/sqlancer/common/gen/PartitionGenerator.java b/src/sqlancer/common/gen/PartitionGenerator.java new file mode 100644 index 000000000..affc62c42 --- /dev/null +++ b/src/sqlancer/common/gen/PartitionGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.common.gen; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.schema.AbstractTableColumn; + +public interface PartitionGenerator, C extends AbstractTableColumn> { + + /** + * Negates a predicate (i.e., uses a NOT operator). + * + * @param predicate + * the boolean predicate. + * + * @return the negated predicate. + */ + E negatePredicate(E predicate); + + /** + * Checks if an expression evaluates to NULL (i.e., implements the IS NULL operator). + * + * @param expr + * the expression + * + * @return an expression that checks whether the expression evaluates to NULL. + */ + E isNull(E expr); +} diff --git a/src/sqlancer/common/gen/TLPWhereGenerator.java b/src/sqlancer/common/gen/TLPWhereGenerator.java new file mode 100644 index 000000000..095a878f9 --- /dev/null +++ b/src/sqlancer/common/gen/TLPWhereGenerator.java @@ -0,0 +1,28 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface TLPWhereGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> + extends PartitionGenerator { + + TLPWhereGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + List generateOrderBys(); +} diff --git a/src/sqlancer/gen/TypedExpressionGenerator.java b/src/sqlancer/common/gen/TypedExpressionGenerator.java similarity index 94% rename from src/sqlancer/gen/TypedExpressionGenerator.java rename to src/sqlancer/common/gen/TypedExpressionGenerator.java index 462643300..3ee7115fe 100644 --- a/src/sqlancer/gen/TypedExpressionGenerator.java +++ b/src/sqlancer/common/gen/TypedExpressionGenerator.java @@ -1,4 +1,4 @@ -package sqlancer.gen; +package sqlancer.common.gen; import java.util.ArrayList; import java.util.Collections; @@ -6,7 +6,7 @@ import sqlancer.Randomly; -public abstract class TypedExpressionGenerator { +public abstract class TypedExpressionGenerator implements ExpressionGenerator { protected List columns = Collections.emptyList(); protected boolean allowAggregates; diff --git a/src/sqlancer/gen/UntypedExpressionGenerator.java b/src/sqlancer/common/gen/UntypedExpressionGenerator.java similarity index 85% rename from src/sqlancer/gen/UntypedExpressionGenerator.java rename to src/sqlancer/common/gen/UntypedExpressionGenerator.java index df14fd030..5a244b53b 100644 --- a/src/sqlancer/gen/UntypedExpressionGenerator.java +++ b/src/sqlancer/common/gen/UntypedExpressionGenerator.java @@ -1,11 +1,11 @@ -package sqlancer.gen; +package sqlancer.common.gen; import java.util.ArrayList; import java.util.List; import sqlancer.Randomly; -public abstract class UntypedExpressionGenerator { +public abstract class UntypedExpressionGenerator implements ExpressionGenerator { protected List columns; protected boolean allowAggregates; @@ -42,7 +42,7 @@ public List generateExpressions(int nr) { return expressions; } - public List generateExpressions(int depth, int nr) { + public List generateExpressions(int nr, int depth) { List expressions = new ArrayList<>(); for (int i = 0; i < nr; i++) { expressions.add(generateExpression(depth)); @@ -63,4 +63,9 @@ public E generateHavingClause() { return expr; } + @Override + public E generatePredicate() { + return generateExpression(); + } + } diff --git a/src/sqlancer/common/log/Loggable.java b/src/sqlancer/common/log/Loggable.java new file mode 100644 index 000000000..7796009ee --- /dev/null +++ b/src/sqlancer/common/log/Loggable.java @@ -0,0 +1,7 @@ +package sqlancer.common.log; + +import java.io.Serializable; + +public interface Loggable extends Serializable { + String getLogString(); +} diff --git a/src/sqlancer/common/log/LoggableFactory.java b/src/sqlancer/common/log/LoggableFactory.java new file mode 100644 index 000000000..5a439714d --- /dev/null +++ b/src/sqlancer/common/log/LoggableFactory.java @@ -0,0 +1,37 @@ +package sqlancer.common.log; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; + +import sqlancer.common.query.Query; + +public abstract class LoggableFactory { + + public Loggable createLoggableWithNoLinebreak(String input) { + return createLoggable(input, ""); + } + + public Loggable createLoggable(String input) { + return createLoggable(input, "\n"); + } + + protected abstract Loggable createLoggable(String input, String suffix); + + public abstract Query getQueryForStateToReproduce(String queryString); + + @Deprecated + public abstract Query commentOutQuery(Query query); + + public Loggable getInfo(String databaseName, String databaseVersion, long seedValue) { + Date date = new Date(); + DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + return infoToLoggable(dateFormat.format(date), databaseName, databaseVersion, seedValue); + } + + protected abstract Loggable infoToLoggable(String time, String databaseName, String databaseVersion, + long seedValue); + + public abstract Loggable convertStacktraceToLoggable(Throwable throwable); + +} diff --git a/src/sqlancer/common/log/LoggedString.java b/src/sqlancer/common/log/LoggedString.java new file mode 100644 index 000000000..696203a78 --- /dev/null +++ b/src/sqlancer/common/log/LoggedString.java @@ -0,0 +1,16 @@ +package sqlancer.common.log; + +public class LoggedString implements Loggable { + private static final long serialVersionUID = 1L; + + private final String loggedString; + + public LoggedString(String loggedString) { + this.loggedString = loggedString; + } + + @Override + public String getLogString() { + return this.loggedString; + } +} diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java new file mode 100644 index 000000000..bdcf2253f --- /dev/null +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -0,0 +1,54 @@ +package sqlancer.common.log; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; + +public class SQLLoggableFactory extends LoggableFactory { + + @Override + protected Loggable createLoggable(String input, String suffix) { + String completeString = input; + if (!input.endsWith(";")) { + completeString += ";"; + } + completeString = completeString.replace("\n", "\\n"); + completeString = completeString.replace("\r", "\\r"); + if (suffix != null && !suffix.isEmpty()) { + completeString += suffix; + } + return new LoggedString(completeString); + } + + @Override + public SQLQueryAdapter getQueryForStateToReproduce(String queryString) { + return new SQLQueryAdapter(queryString); + } + + @Override + public SQLQueryAdapter commentOutQuery(Query query) { + String queryString = query.getLogString(); + String newQueryString = "-- " + queryString; + return new SQLQueryAdapter(newQueryString); + } + + @Override + protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { + StringBuilder sb = new StringBuilder(); + sb.append("-- Time: ").append(time).append("\n"); + sb.append("-- Database: ").append(databaseName).append("\n"); + sb.append("-- Database version: ").append(databaseVersion).append("\n"); + sb.append("-- seed value: ").append(seedValue).append("\n"); + return new LoggedString(sb.toString()); + } + + @Override + public Loggable convertStacktraceToLoggable(Throwable throwable) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + throwable.printStackTrace(pw); + return new LoggedString("--" + sw.toString().replace("\n", "\n--")); + } +} diff --git a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java new file mode 100644 index 000000000..582a19c1d --- /dev/null +++ b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java @@ -0,0 +1,107 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; + +/** + * Shared skeleton for the {@link Reproducer}s of oracles that detect a bug by comparing two evaluations of a + * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}). Reduction re-runs + * both sides against the reduced database and reports whether they still disagree. + * + *

+ * The separate case where the original bug was an unexpected DBMS error rather than a mismatch is handled by + * {@link UnexpectedErrorReproducer}, so a subclass here deals only with comparing two sides and never with error + * handling. + * + *

+ * This class owns the compare-and-report control flow and the {@link #getBugInformation()} header. Subclasses supply + * how each side is evaluated, how the two are compared, and how the failing queries are rendered in the reduced test + * case. + * + * @param + * the DBMS-specific global state class + * @param + * the type each side evaluates to (e.g. a result set as a list of strings, a row count) + */ +public abstract class AbstractComparisonReproducer, R> implements Reproducer { + + /** + * Evaluates the original side against the reduced database. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @return the original side's value + * + * @throws SQLException + * if a DBMS interaction fails + */ + protected abstract R evaluateOriginal(G globalState) throws SQLException; + + /** + * Evaluates the transformed side against the reduced database. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @return the transformed side's value + * + * @throws SQLException + * if a DBMS interaction fails + */ + protected abstract R evaluateTransformed(G globalState) throws SQLException; + + /** + * Whether the two evaluated sides disagree in the way that constitutes the bug. + * + * @param original + * the original side's value + * @param transformed + * the transformed side's value + * @param globalState + * the state the sides were evaluated against + * + * @return {@code true} if the sides differ (i.e. the bug still triggers) + */ + protected abstract boolean sidesDiffer(R original, R transformed, G globalState); + + @Override + public final boolean bugStillTriggers(G globalState) { + R original; + R transformed; + try { + original = evaluateOriginal(globalState); + transformed = evaluateTransformed(globalState); + } catch (AssertionError | SQLException | RuntimeException e) { + // any failure re-running the two sides means this reduced database no longer shows the mismatch + return false; + } + return sidesDiffer(original, transformed, globalState); + } + + @Override + public final String getBugInformation() { + StringBuilder sb = new StringBuilder(); + sb.append(mismatchHeaderLine()).append(System.lineSeparator()); + appendQueryLines(sb); + return sb.toString(); + } + + /** + * The header line (without trailing line separator) describing the mismatch. For example, "-- On the database set + * up by the statements above, the result sets of the following queries mismatch:". + * + * @return the mismatch header line + */ + protected abstract String mismatchHeaderLine(); + + /** + * Appends the failing queries to {@code sb}, one commented line each, so the reduced test case is self-contained. + * + * @param sb + * the builder to append to + */ + protected abstract void appendQueryLines(StringBuilder sb); +} diff --git a/src/sqlancer/common/oracle/CERTOracle.java b/src/sqlancer/common/oracle/CERTOracle.java new file mode 100644 index 000000000..48c174ef2 --- /dev/null +++ b/src/sqlancer/common/oracle/CERTOracle.java @@ -0,0 +1,133 @@ +package sqlancer.common.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class CERTOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private final CheckedFunction> rowCountParser; + private final CheckedFunction> queryPlanParser; + + private CERTGenerator gen; + private final ExpectedErrors errors; + + public CERTOracle(G state, CERTGenerator gen, ExpectedErrors expectedErrors, + CheckedFunction> rowCountParser, + CheckedFunction> queryPlanParser) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + this.rowCountParser = rowCountParser; + this.queryPlanParser = queryPlanParser; + } + + @Override + public void check() throws SQLException { + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + List fetchColumns = gen.generateFetchColumns(false); + + Z select = gen.generateSelect(); + select.setFetchColumns(fetchColumns); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateBooleanExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByClause(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateBooleanExpression()); + } + } + + List queryPlan1Sequences = new ArrayList<>(); + List queryPlan2Sequences = new ArrayList<>(); + + String queryString1 = gen.generateExplainQuery(select); + long rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + boolean increase = gen.mutate(select); + String queryString2 = gen.generateExplainQuery(select); + long rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check the results + if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { + throw new AssertionError("Inconsistent result for query: " + queryString1 + "; --" + rowCount1 + "\n" + + queryString2 + "; --" + rowCount2); + } + } + + private Long getRow(SQLGlobalState globalState, String explainQuery, List queryPlanSequences) + throws AssertionError, SQLException { + Optional row = Optional.empty(); + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + Optional rowCount = rowCountParser.apply(rs); + if (row.isEmpty() && rowCount.isPresent()) { + row = rowCount; + } + + Optional queryPlanSequence = queryPlanParser.apply(rs); + queryPlanSequence.ifPresent(qps -> queryPlanSequences.add(qps)); + } + } + } catch (IgnoreMeException e) { + throw new IgnoreMeException(); + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + + return row.orElseThrow(IgnoreMeException::new); + } + + @FunctionalInterface + public interface CheckedFunction { + R apply(T t) throws SQLException; + } +} diff --git a/src/sqlancer/common/oracle/CERTOracleBase.java b/src/sqlancer/common/oracle/CERTOracleBase.java new file mode 100644 index 000000000..42e8e5833 --- /dev/null +++ b/src/sqlancer/common/oracle/CERTOracleBase.java @@ -0,0 +1,88 @@ +package sqlancer.common.oracle; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; + +public abstract class CERTOracleBase> implements TestOracle { + + protected final S state; + protected final ExpectedErrors errors; + protected List queryPlan1Sequences; + protected List queryPlan2Sequences; + + protected enum Mutator { + JOIN, DISTINCT, WHERE, GROUPBY, HAVING, AND, OR, LIMIT; + + public static Mutator getRandomExcept(Mutator... exclude) { + Mutator[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(Mutator[]::new); + return Randomly.fromOptions(values); + } + } + + protected CERTOracleBase(S state) { + this.state = state; + this.errors = new ExpectedErrors(); + } + + protected boolean mutate(Mutator... exclude) { + Mutator m = Mutator.getRandomExcept(exclude); + switch (m) { + case JOIN: + return mutateJoin(); + case DISTINCT: + return mutateDistinct(); + case WHERE: + return mutateWhere(); + case GROUPBY: + return mutateGroupBy(); + case HAVING: + return mutateHaving(); + case AND: + return mutateAnd(); + case OR: + return mutateOr(); + case LIMIT: + return mutateLimit(); + default: + throw new AssertionError(m); + } + } + + protected boolean mutateJoin() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateDistinct() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateWhere() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateGroupBy() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateHaving() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateAnd() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateOr() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateLimit() { + throw new UnsupportedOperationException(); + } + +} diff --git a/src/sqlancer/common/oracle/CODDTestBase.java b/src/sqlancer/common/oracle/CODDTestBase.java new file mode 100644 index 000000000..639a4f077 --- /dev/null +++ b/src/sqlancer/common/oracle/CODDTestBase.java @@ -0,0 +1,25 @@ +package sqlancer.common.oracle; + +import sqlancer.Main.StateLogger; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; + +public abstract class CODDTestBase> implements TestOracle { + protected final S state; + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + protected String auxiliaryQueryString; + protected String foldedQueryString; + protected String originalQueryString; + + public CODDTestBase(S state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } +} diff --git a/src/sqlancer/common/oracle/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java new file mode 100644 index 000000000..e2f96f785 --- /dev/null +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -0,0 +1,37 @@ +package sqlancer.common.oracle; + +import java.util.List; + +import sqlancer.GlobalState; + +public class CompositeTestOracle> implements TestOracle { + + private final List> oracles; + private final G globalState; + private int i; + private int iLast; + + public CompositeTestOracle(List> oracles, G globalState) { + this.globalState = globalState; + this.oracles = oracles; + } + + @Override + public void check() throws Exception { + try { + oracles.get(i).check(); + iLast = i; + boolean lastOracleIndex = i == oracles.size() - 1; + if (!lastOracleIndex) { + globalState.getManager().incrementSelectQueryCount(); + } + } finally { + i = (i + 1) % oracles.size(); + } + } + + @Override + public String getLastQueryString() { + return oracles.get(iLast).getLastQueryString(); + } +} diff --git a/src/sqlancer/common/oracle/DQEBase.java b/src/sqlancer/common/oracle/DQEBase.java new file mode 100644 index 000000000..0d954b115 --- /dev/null +++ b/src/sqlancer/common/oracle/DQEBase.java @@ -0,0 +1,145 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryError; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractTables; + +/* + * In DBMSs, SELECT, UPDATE and DELETE queries utilize predicates (i.e., WHERE clauses) to specify which rows to retrieve, update or delete, respectively. + * If they use the same predicate φ, they should access the same rows in a database. + * Ideally, DBMSs can adopt the same implementations for predicate evaluation in SELECT, UPDATE and DELETE queries. + * However, a DBMS usually adopts different implementations for predicate evaluation in SELECT, UPDATE and DELETE queries due to various optimization choices. + * Inconsistent implementations for predicate evaluation among these queries can cause SELECT, UPDATE and DELETE queries with the same predicate φ to access different rows. + * + * + * Inspired by this key observation, we propose Differential Query Execution(DQE), a novel and general approach to detect logic bugs in SELECT, UPDATE and DELETE queries. + * DQE solves the test oracle problem by executing SELECT, UPDATE and DELETE queries with the same predicate φ, and observing inconsistencies among their execution results. + * For example, if a row that is updated by an UPDATE query with a predicate φ does not appear in the query result of a SELECT query with the same predicate φ, a logic bug is detected in the target DBMS. + * The key challenge of DQE is to automatically obtain the accessed rows for a given SELECT, UPDATE or DELETE query. + * To address this challenge, we append two extra columns to each table in a database, to uniquely identify each row and track whether a row has been modified, respectively. + * We further rewrite SELECT and UPDATE queries to identify their accessed rows. + * + * more information see [DQE paper](https://ieeexplore.ieee.org/document/10172736) + */ + +public abstract class DQEBase> { + + public static final String COLUMN_ROWID = "rowId"; + public static final String COLUMN_UPDATED = "updated"; + + protected final S state; + protected final ExpectedErrors selectExpectedErrors = new ExpectedErrors(); + protected final ExpectedErrors updateExpectedErrors = new ExpectedErrors(); + protected final ExpectedErrors deleteExpectedErrors = new ExpectedErrors(); + + protected final Main.StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + + public DQEBase(S state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } + + public abstract String generateSelectStatement(AbstractTables tables, String tableName, + String whereClauseStr); + + public abstract String generateUpdateStatement(AbstractTables tables, String tableName, + String whereClauseStr); + + public abstract String generateDeleteStatement(String tableName, String whereClauseStr); + + // Add auxiliary columns to the database A abstract method, subclasses need to implement it. + public abstract void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException; + + public void dropAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { + String tableName = table.getName(); + String dropColumnRowId = String.format("ALTER TABLE %s DROP COLUMN %s", tableName, COLUMN_ROWID); + new SQLQueryAdapter(dropColumnRowId).execute(state); + String dropColumnUpdated = String.format("ALTER TABLE %s DROP COLUMN %s", tableName, COLUMN_UPDATED); + new SQLQueryAdapter(dropColumnUpdated).execute(state); + } + + // This interface is to record Error code + public interface UpdateErrorCodes { + + } + + public interface ErrorCodeStrategy { + Set getUpdateSpecificErrorCodes(); + + Set getDeleteSpecificErrorCodes(); + + } + + /** + * The core idea of DQE is that the SELECT, UPDATE and DELETE queries with the same predicate φ should access the + * same rows. If these queries access different rows, DQE reveals a potential logic bug in the target DBMS. + */ + public static class SQLQueryResult { + + private final Map, Set> accessedRows; // Table name with respect rows + private final List queryErrors; + + public SQLQueryResult(Map, Set> accessedRows, + List queryErrors) { + this.accessedRows = accessedRows; + this.queryErrors = queryErrors; + } + + public Map, Set> getAccessedRows() { + return accessedRows; + } + + public List getQueryErrors() { + return queryErrors; + } + + public boolean hasEmptyErrors() { + return queryErrors.isEmpty(); + } + + public boolean hasSameErrors(SQLQueryResult that) { + if (queryErrors.size() != that.getQueryErrors().size()) { + return false; + } else { + for (int i = 0; i < queryErrors.size(); i++) { + if (!queryErrors.get(i).equals(that.getQueryErrors().get(i))) { + return false; + } + } + } + return true; + } + + public boolean hasAccessedRows() { + if (accessedRows.isEmpty()) { + return false; + } + for (Set accessedRow : accessedRows.values()) { + if (!accessedRow.isEmpty()) { + return true; + } + } + return false; + } + + public boolean hasSameAccessedRows(SQLQueryResult that) { + return accessedRows.equals(that.getAccessedRows()); + } + + } +} diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java new file mode 100644 index 000000000..77a8b528a --- /dev/null +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -0,0 +1,532 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.gen.EETDMLGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle for DML statements, based on "Detecting Logic Bugs in Database + * Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * Whereas {@link EETOracle} transforms a SELECT and compares the two result sets, this oracle transforms a DML + * statement and compares the two database states produced. + * + *

+ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which + * uniquely identifies each row, and each statement is executed inside a transaction that is rolled back, so the two + * statements can be compared against the same starting state without permanently modifying the database. The state is + * captured as a full post-image: each surviving row's identifier together with its content column values, ordered by + * the identifier. This single value-level surface covers every DML statement — a DELETE removes rows from it, an UPDATE + * changes values in it, an INSERT adds rows to it (row identity alone would suffice for DELETE, but not for UPDATE, + * which also transforms the written values). Because rolling back a statement requires a transactional storage engine, + * the DBMS-specific setup must ensure only such engines are used while this oracle is active. + * + *

+ * DELETE, UPDATE and INSERT are currently supported (one is chosen at random per check). INSERT uses the + * {@code INSERT ... SELECT} form so its transformed value expressions may reference columns; each inserted row is given + * a deterministic identifier derived from its source row so the two runs' post-images align. To support reduction, a + * {@link Reproducer} replays the whole comparison (adding and stamping the row-identifier column, running both + * statements in rolled-back transactions and comparing the post-images) against the reduced database. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class + */ +public class EETDMLOracle, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETDMLGenerator gen; + private final EETTransformer transformer; + private final ExpectedErrors errors; + + private static final int MAX_DIFF_ROWS_REPORTED = 10; // max differing post-image rows displayed in report log + private String generatedQueryString; + private Reproducer reproducer; + + // The SQL and metadata to run and observe one DML comparison, captured as strings so a reproducer can replay it + // against a reduced database without the generator or live schema objects. + private static final class ComparisonQueries { + private final String originalStatement; + private final String transformedStatement; + private final String addRowIdColumn; + private final String stampRowIds; + private final String beginTransaction; + private final String rollback; + private final String dropRowIdColumn; + private final String selectPostImage; + private final int columnCount; + + ComparisonQueries(String originalStatement, String transformedStatement, String addRowIdColumn, + String stampRowIds, String beginTransaction, String rollback, String dropRowIdColumn, + String selectPostImage, int columnCount) { + this.originalStatement = originalStatement; + this.transformedStatement = transformedStatement; + this.addRowIdColumn = addRowIdColumn; + this.stampRowIds = stampRowIds; + this.beginTransaction = beginTransaction; + this.rollback = rollback; + this.dropRowIdColumn = dropRowIdColumn; + this.selectPostImage = selectPostImage; + this.columnCount = columnCount; + } + } + + // The post-images the original and transformed statements produced, compared for equality to detect the bug. + private static final class PostImages { + private final List> original; + private final List> transformed; + + PostImages(List> original, List> transformed) { + this.original = original; + this.transformed = transformed; + } + } + + // Reproduces a post-image mismatch against the reduced database. Unlike EETOracle's comparison reproducer this does + // not extend AbstractComparisonReproducer: the two sides are not independent, because the row-id stamping (UUID()) + // must run once so both observe the same rows, so both post-images are computed together. + private final class EETDMLReproducer implements Reproducer { + private final ComparisonQueries queries; + + EETDMLReproducer(ComparisonQueries queries) { + this.queries = queries; + } + + @Override + public boolean bugStillTriggers(G globalState) { + PostImages images; + try { + images = computePostImages(globalState, queries); + } catch (AssertionError | SQLException | RuntimeException e) { + // any failure re-running the comparison means this reduced database no longer shows the mismatch + return false; + } + return !images.original.equals(images.transformed); + } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + sb.append("-- On the database set up by the statements above, the following statements leave the database" + + " in different states:").append(System.lineSeparator()); + renderStatementLines(sb, queries); + return sb.toString(); + } + } + + // Builds the reproducer for an unexpected DBMS error, which replays the whole comparison and checks the same error + // still fires. + private UnexpectedErrorReproducer errorReproducer(ComparisonQueries queries, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> computePostImages(globalState, queries); + StringBuilder sb = new StringBuilder(); + renderStatementLines(sb, queries); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + + // Renders the failing statements as commented lines, shared by the mismatch and the unexpected-error reproducers. + private static void renderStatementLines(StringBuilder sb, ComparisonQueries queries) { + sb.append("-- original: ").append(queries.originalStatement).append(';').append(System.lineSeparator()); + sb.append("-- transformed: ").append(queries.transformedStatement).append(';').append(System.lineSeparator()); + } + + public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.transformer = gen.createTransformer(); + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + reproducer = null; + List tables = state.getSchema().getDatabaseTables(); + if (tables.isEmpty()) { + throw new IgnoreMeException(); + } + // A DML statement targets a single table, so operate on exactly one; confining the generator to it keeps the + // predicate and value expressions from referencing another table's columns (which would render invalid + // single-table DML). + T table = Randomly.fromList(tables); + gen = gen.setTablesAndColumns(new AbstractTables<>(List.of(table))); + + E predicate = gen.generateBooleanExpression(); + // The WHERE predicate is evaluated in a boolean context. + E transformedPredicate = transformer.transform(predicate, true); + + // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total + // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped + // row set is deterministic and equal across the runs while still exercising varied orderings. + Integer limit = null; + List orderByColumns = List.of(); + if (Randomly.getBoolean()) { + limit = (int) Randomly.getNotCachedInteger(0, 10); + orderByColumns = Randomly.subset(table.getColumns()); + } + + // Generators for the different kinds of statement this oracle supports. One is chosen at random per check + List> statementGenerators = List.of(this::generateDeleteStatements, + this::generateUpdateStatements, this::generateInsertStatements); + StatementPair statements = Randomly.fromList(statementGenerators).generate(table, predicate, + transformedPredicate, orderByColumns, limit); + String originalStatement = statements.original; + String transformedStatement = statements.transformed; + generatedQueryString = originalStatement; + + // Capture, as strings, everything needed to run and observe this comparison: the two statements plus the + // auxiliary-column setup, per-run snapshot and teardown. A reproducer replays these against a reduced database, + // where the live generator and schema objects no longer apply. + ComparisonQueries queries = new ComparisonQueries(originalStatement, transformedStatement, + gen.addRowIdColumnStatement(table), gen.stampRowIdsStatement(table), gen.beginTransactionStatement(), + gen.rollbackTransactionStatement(), gen.dropRowIdColumnStatement(table), + gen.selectPostImageStatement(table), gen.postImageColumns(table).size()); + + PostImages images; + try { + images = computePostImages(state, queries); + } catch (AssertionError unexpectedError) { + reproducer = errorReproducer(queries, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + reproducer = new EETDMLReproducer(queries); + if (!images.original.equals(images.transformed)) { + throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, images.original, + images.transformed)); + } + } + + /** + * Runs the whole comparison against {@code globalState}: adds and stamps the row-identifier column once (so both + * runs observe the same rows), snapshots the post-image each statement produces (each inside a rolled-back + * transaction), and drops the column. Both {@link #check()} and the reproducers call this, the former against the + * live database and the latter against a reduced one. A DBMS error the oracle tolerates aborts with + * {@link IgnoreMeException}; an oracle logic bug or unexpected error surfaces as {@link AssertionError}. + * + * @param globalState + * the state whose connection the comparison runs against + * @param queries + * the statements and auxiliary SQL to run + * + * @return the post-images the original and transformed statements produced + * + * @throws SQLException + * if a DBMS interaction fails + */ + private PostImages computePostImages(G globalState, ComparisonQueries queries) throws SQLException { + // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the + // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the + // column and cause cascading duplicate-column failures + if (!new SQLQueryAdapter(queries.addRowIdColumn, errors, true).execute(globalState)) { + throw new IgnoreMeException(); + } + try { + // Stamp identifiers once, in autocommit mode, before both runs: both then observe the same rows. + if (!new SQLQueryAdapter(queries.stampRowIds, errors).execute(globalState)) { + throw new IgnoreMeException(); + } + List> original = snapshotSide(globalState, queries.originalStatement, queries); + List> transformed = snapshotSide(globalState, queries.transformedStatement, queries); + return new PostImages(original, transformed); + } finally { + new SQLQueryAdapter(queries.dropRowIdColumn, errors, true).execute(globalState); + } + } + + /** + * Generates a DML statement of one kind together with its transformed counterpart. The kinds share this signature + * so the oracle can pick one of them at random per check. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ + @FunctionalInterface + private interface DMLStatementGenerator { + StatementPair generate(T table, E predicate, E transformedPredicate, List orderByColumns, Integer limit); + } + + /** + * A DML statement and its transformed counterpart, which must leave the database in the same state. + */ + private static final class StatementPair { + private final String original; + private final String transformed; + + StatementPair(String original, String transformed) { + this.original = original; + this.transformed = transformed; + } + } + + /** + * Generates an UPDATE and its transformed counterpart. Besides the WHERE predicate, UPDATE also transforms the + * written values: each SET value expression is transformed in a scalar context. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the statement, empty if it is not capped by a limit + * @param limit + * the maximum number of rows to modify, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateUpdateStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + List> assignments = gen.generateSetAssignments(); + List> transformedAssignments = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + E transformedValue = transformer.transform(assignment.getValue(), false); + transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); + } + return new StatementPair(gen.updateStatement(table, assignments, predicate, orderByColumns, limit), + gen.updateStatement(table, transformedAssignments, transformedPredicate, orderByColumns, limit)); + } + + /** + * Generates a DELETE and its transformed counterpart, which differ only in their WHERE predicate. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the statement, empty if it is not capped by a limit + * @param limit + * the maximum number of rows to modify, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateDeleteStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + return new StatementPair(gen.deleteStatement(table, predicate, orderByColumns, limit), + gen.deleteStatement(table, transformedPredicate, orderByColumns, limit)); + } + + /** + * Generates an {@code INSERT ... SELECT} and its transformed counterpart. Besides the WHERE predicate, which + * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. + * + *

+ * The ordering and limit cap the source rows the statement reads, so it inserts one row per source row kept. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the source rows, empty if the statement is not capped by a limit + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + List values = gen.generateInsertValues(); + List transformedValues = new ArrayList<>(); + for (E value : values) { + transformedValues.add(transformer.transform(value, false)); + } + boolean withPredicate = Randomly.getBoolean(); + return new StatementPair( + gen.insertStatement(table, values, withPredicate ? predicate : null, orderByColumns, limit), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null, + orderByColumns, limit)); + } + + /** + * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: + * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A + * DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug or unexpected error + * surfaces as {@link AssertionError}. + * + * @param globalState + * the state whose connection the statement runs against + * @param statement + * the DML statement to execute + * @param queries + * supplies the transaction control and post-image select SQL and the post-image's column count + * + * @return the post-image, as one string list (identifier followed by content column values) per surviving row + * + * @throws SQLException + * if a DBMS interaction other than running {@code statement} fails; an error from {@code statement} + * itself instead surfaces as {@link IgnoreMeException} or {@link AssertionError} + */ + private List> snapshotSide(G globalState, String statement, ComparisonQueries queries) + throws SQLException { + new SQLQueryAdapter(queries.beginTransaction).execute(globalState); + try { + // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. + boolean succeeded = new SQLQueryAdapter(statement, errors).execute(globalState); + if (!succeeded) { + // The statement hit an error the oracle tolerates; do not compare states (as EETOracle does for + // SELECT). + throw new IgnoreMeException(); + } + return snapshotPostImage(globalState, queries.selectPostImage, queries.columnCount); + } finally { + new SQLQueryAdapter(queries.rollback).execute(globalState); + } + } + + /** + * Reads the post-image produced by {@code selectStatement} into one string list per row (each column via + * {@code getString}). A DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug + * or unexpected error surfaces as {@link AssertionError}. + * + * @param globalState + * the state whose connection the select runs against + * @param selectStatement + * the post-image select to read; its columns are the identifier followed by the content columns + * @param columnCount + * the number of columns to read from each row + * + * @return the read rows, in the select's order + * + * @throws SQLException + * if cleanup fails (errors thrown elsewhere will always be rethrown as {@link IgnoreMeException} or + * {@link AssertionError}) + */ + private List> snapshotPostImage(G globalState, String selectStatement, int columnCount) + throws SQLException { + List> rows = new ArrayList<>(); + SQLQueryAdapter q = new SQLQueryAdapter(selectStatement, errors, true, + globalState.getOptions().canonicalizeSqlString()); + SQLancerResultSet result = null; + try { + result = q.executeAndGet(globalState); + if (result == null) { + throw new IgnoreMeException(); + } + while (result.next()) { + List row = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + row.add(result.getString(i)); + } + rows.add(row); + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw e; + } + Throwable current = e; + while (current != null) { + if (current.getMessage() != null && errors.errorIsExpected(current.getMessage())) { + throw new IgnoreMeException(); + } + current = current.getCause(); + } + throw new AssertionError(selectStatement, e); + } finally { + if (result != null && !result.isClosed()) { + result.close(); + } + } + return rows; + } + + private String mismatchMessage(T table, String originalStatement, String transformedStatement, + List> originalImage, List> transformedImage) { + List header = gen.postImageColumns(table); + // Where the identifier sits within a post-image row, per the layout the generator defines + int rowIdIndex = header.indexOf(EETDMLGenerator.ROW_ID_COLUMN); + + Map> originalByRowId = indexByRowId(originalImage, rowIdIndex); + Map> transformedByRowId = indexByRowId(transformedImage, rowIdIndex); + Set allRowIds = new TreeSet<>(); + allRowIds.addAll(originalByRowId.keySet()); + allRowIds.addAll(transformedByRowId.keySet()); + + String nl = System.lineSeparator(); + StringBuilder message = new StringBuilder() + .append("-- The original and transformed statements left the database in different states.").append(nl) + .append("-- original: ").append(originalStatement).append(';').append(nl).append("-- transformed: ") + .append(transformedStatement).append(';').append(nl).append("-- differing post-image rows (") + .append(String.join(", ", header)).append("):").append(nl); + int shown = 0; + for (String rowId : allRowIds) { + List originalRow = originalByRowId.get(rowId); + List transformedRow = transformedByRowId.get(rowId); + if (Objects.equals(originalRow, transformedRow)) { + continue; + } + if (shown == MAX_DIFF_ROWS_REPORTED) { + message.append("-- ... (further differences omitted)").append(nl); + break; + } + message.append("-- original: ").append(renderRow(originalRow)).append(nl); + message.append("-- transformed: ").append(renderRow(transformedRow)).append(nl); + shown++; + } + return message.toString(); + } + + // Indexes a post-image by its row identifier, which each row holds at rowIdIndex + private static Map> indexByRowId(List> image, int rowIdIndex) { + Map> byRowId = new LinkedHashMap<>(); + for (List row : image) { + byRowId.put(row.get(rowIdIndex), row); + } + return byRowId; + } + + // Renders a post-image row for the finding message, or "(row absent)" when the row is missing on that side + private static String renderRow(List row) { + return row == null ? "(row absent)" : row.toString(); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java new file mode 100644 index 000000000..0cd4ae99e --- /dev/null +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -0,0 +1,199 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.EETGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle, based on "Detecting Logic Bugs in Database Engines via Equivalent + * Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * The oracle generates a random query and then transforms its expressions (the WHERE predicate and the fetch columns) + * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation + * preserves semantics, the original and the transformed query must return the same result set; any discrepancy + * indicates a logic bug in the DBMS. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class + */ +public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETGenerator gen; + private final EETTransformer transformer; + private final ExpectedErrors errors; + + private Reproducer reproducer; + private String generatedQueryString; + + private final class EETReproducer extends AbstractComparisonReproducer> { + private final String originalQueryString; + private final String transformedQueryString; + + EETReproducer(String originalQueryString, String transformedQueryString) { + this.originalQueryString = originalQueryString; + this.transformedQueryString = transformedQueryString; + } + + @Override + protected List evaluateOriginal(G globalState) throws SQLException { + // Re-execute against the current (reduced) database instead of comparing against a cached result set, + // which would be stale once statements have been removed. + return ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + } + + @Override + protected List evaluateTransformed(G globalState) throws SQLException { + return ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); + } + + @Override + protected boolean sidesDiffer(List original, List transformed, G globalState) { + try { + ComparatorHelper.assumeResultSetsAreEqual(original, transformed, originalQueryString, + List.of(transformedQueryString), globalState); + } catch (AssertionError resultSetMismatch) { + return true; + } + return false; + } + + @Override + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { + renderQueryLines(sb, originalQueryString, transformedQueryString); + } + } + + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. + // transformedQueryString is null when the error struck the original query before any transformation existed. + private static void renderQueryLines(StringBuilder sb, String originalQueryString, String transformedQueryString) { + sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (transformedQueryString != null) { + sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); + } + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs the query (or both queries) and checks the same + // error still fires. transformedQueryString is null when only the original query ran before the error. + private UnexpectedErrorReproducer errorReproducer(String originalQueryString, String transformedQueryString, + String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + if (transformedQueryString != null) { + ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); + } + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, originalQueryString, transformedQueryString); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + + public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.transformer = gen.createTransformer(); + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + reproducer = null; + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + Z select = gen.generateSelect(); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + List fetchColumns = gen.generateFetchColumns(true); + select.setFetchColumns(fetchColumns); + E whereClause = gen.generateBooleanExpression(); + select.setWhereClause(whereClause); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List originalResultSet; + try { + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + // an unexpected DBMS error on the original query alone is itself a bug worth reducing; + // there is no transformed query yet, so only the original is replayed + reproducer = errorReproducer(originalQueryString, null, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, + // while the WHERE clause is evaluated in a boolean context. + List transformedFetchColumns = fetchColumns.stream().map(c -> transformer.transform(c, false)) + .collect(Collectors.toList()); + select.setFetchColumns(transformedFetchColumns); + select.setWhereClause(transformer.transform(whereClause, true)); + + String transformedQueryString = select.asString(); + List transformedResultSet; + try { + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + state); + } catch (AssertionError unexpectedError) { + // the semantics-preserving transformation made the query trigger a DBMS error that the + // original did not, which is a bug worth reducing + reproducer = errorReproducer(originalQueryString, transformedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is + // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. + reproducer = new EETReproducer(originalQueryString, transformedQueryString); + + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), state); + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java new file mode 100644 index 000000000..b472b8aff --- /dev/null +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -0,0 +1,360 @@ +package sqlancer.common.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; + +/** + * Abstract base class for EET (Equivalent Expression Transformation) tree-walkers, based on "Detecting Logic Bugs in + * Database Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework + * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST + * nodes from their transformed children, the abstract factory methods to construct new nodes, and the type hooks + * ({@link #inferType} and {@link #generateExpressionOfType}) that realize the paper's {@code rand_expr(type(expr))}; + * everything else (the rule logic, context threading, and tree-walking orchestration) is provided here. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain used by {@link #inferType} and {@link #generateExpressionOfType} + */ +public abstract class EETTransformer, T> { + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE + private E trueExpr() { + E p = generateBooleanExpression(); + return orExpr(orExpr(p, not(p)), isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE + private E falseExpr() { + E p = generateBooleanExpression(); + return and(and(p, not(p)), isNotNull(p)); + } + + /** + * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of + * {@code expr}. Although the generated expression is never evaluated (it occupies the redundant branch of rules 3 + * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could + * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to + * {@code expr} itself, which trivially has the correct type (degenerating to rules 5 and 6). + * + * @param expr + * the expression whose static type the generated expression must match + * + * @return a random expression whose static type matches that of {@code expr} + */ + private E randExprOfSameType(E expr) { + T type = inferType(expr); + if (type == null) { + return expr; + } + return generateExpressionOfType(type); + } + + /** + * The first six transformation rules of the EET paper (Table 2). Each rule knows how to apply itself + * ({@link #apply}) and in which contexts it preserves the expression's value ({@link #isApplicable}). Rule No. 7 + * (transform the expression to itself) is not modelled here: it is the fallback applied by {@link #applyRandomRule} + * when no other rule is applicable. + */ + private enum Rule { + // expr => false_expr OR expr + RULE_1 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.orExpr(t.falseExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => true_expr AND expr + RULE_2 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.and(t.trueExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END + RULE_3 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.falseExpr(), t.randExprOfSameType(expr), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END + RULE_4 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.trueExpr(), expr, t.randExprOfSameType(expr)); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END + RULE_5 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END + RULE_6 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }; + + /** + * Applies this rule to {@code expr}, producing a semantically equivalent expression. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain + * @param t + * the transformer providing the DBMS-specific node factories + * @param expr + * the expression to transform + * + * @return a semantically equivalent expression + */ + abstract , T> E apply(EETTransformer t, E expr); + + /** + * Whether this rule preserves {@code expr}'s value in the given context. + * + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (rules 1 and 2 are only applicable if + * this is the case) + * @param caseWhenApplicable + * whether {@code expr} may be wrapped in a CASE WHEN expression + * + * @return {@code true} if this rule preserves {@code expr}'s value in the given context + */ + abstract boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable); + } + + /** + * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent + * expression. When no rule is applicable, {@code expr} is returned unchanged (rule No. 7 of the EET paper). + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * + * @return a semantically equivalent expression + */ + protected E applyRandomRule(E expr, boolean booleanContext) { + boolean caseWhenApplicable = isCaseWhenApplicable(expr); + List applicableRules = new ArrayList<>(); + for (Rule rule : Rule.values()) { + if (rule.isApplicable(booleanContext, caseWhenApplicable)) { + applicableRules.add(rule); + } + } + if (applicableRules.isEmpty()) { + return expr; // rule 7 fallback: transform expression to itself + } + return Randomly.fromList(applicableRules).apply(this, expr); + } + + /** + * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the + * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * + * @return a semantically equivalent expression + */ + public E transform(E expr, boolean booleanContext) { + return transformNode(expr, booleanContext, true); + } + + /** + * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * @param forceApply + * whether a rule must be applied at this node rather than only with some probability + * + * @return the transformed expression + */ + protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { + E descended = descend(expr, booleanContext); + if (forceApply || Randomly.getBoolean()) { + return applyRandomRule(descended, booleanContext); + } + return descended; + } + + /** + * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each + * child. Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable + * transformation will still be applied to them by the calling {@link #transformNode}. + * + * @param expr + * the expression to descend into + * @param booleanContext + * the context in which {@code expr} itself is evaluated (used to determine child contexts) + * + * @return a rebuilt copy of {@code expr} with transformed children, or {@code expr} itself if it is a leaf + */ + protected abstract E descend(E expr, boolean booleanContext); + + /** + * Builds {@code left AND right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left AND right} expression + */ + protected abstract E and(E left, E right); + + /** + * Builds {@code left OR right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left OR right} expression + */ + protected abstract E orExpr(E left, E right); + + /** + * Builds {@code NOT expr}. + * + * @param expr + * the operand + * + * @return the {@code NOT expr} expression + */ + protected abstract E not(E expr); + + /** + * Builds {@code expr IS NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NULL} expression + */ + protected abstract E isNull(E expr); + + /** + * Builds {@code expr IS NOT NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NOT NULL} expression + */ + protected abstract E isNotNull(E expr); + + /** + * Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. + * + * @param condition + * the WHEN condition + * @param thenExpr + * the THEN branch + * @param elseExpr + * the ELSE branch + * + * @return the CASE WHEN expression + */ + protected abstract E caseWhen(E condition, E thenExpr, E elseExpr); + + /** + * Generates a fresh random boolean expression, reusing the variables available to the query generator. + * + * @return a fresh random boolean expression + */ + protected abstract E generateBooleanExpression(); + + /** + * Infers the static type of {@code expr}, or returns {@code null} if it cannot be determined. The type domain + * {@code T} is DBMS-specific and may be coarse: it only needs to be precise enough that replacing an expression + * with another of the same {@code T} leaves the DBMS's CASE WHEN result-type resolution unaffected. Returning + * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead branch. + * Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is + * uncertain. + * + * @param expr + * the expression whose static type is to be inferred + * + * @return the inferred static type of {@code expr}, or {@code null} if it cannot be determined + */ + protected abstract T inferType(E expr); + + /** + * Generates a fresh random expression of static type {@code type}, reusing the variables available to the query + * generator. DBMSs with a typed expression generator can delegate to it directly; DBMSs with an untyped generator + * can instead wrap an arbitrary random expression in a CAST to {@code type} (which requires every value of + * {@code T} to be a valid CAST target). + * + * @param type + * the static type the generated expression must have + * + * @return a fresh random expression of static type {@code type} + */ + protected abstract E generateExpressionOfType(T type); + + /** + * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not + * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + * + * @param expr + * the expression to test + * + * @return {@code true} if {@code expr} may be wrapped in a CASE WHEN expression + */ + protected abstract boolean isCaseWhenApplicable(E expr); +} diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java new file mode 100644 index 000000000..ba48e80bb --- /dev/null +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -0,0 +1,227 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.function.Function; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class NoRECOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + + private NoRECGenerator gen; + private final ExpectedErrors errors; + + private Reproducer reproducer; + private String lastQueryString; + + private static class NoRECReproducer> + extends AbstractComparisonReproducer { + private final Function optimizedQuery; + private final Function unoptimizedQuery; + private final String optimizedQueryString; + private final String unoptimizedQueryString; + + NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, + String optimizedQueryString, String unoptimizedQueryString) { + this.optimizedQuery = optimizedQuery; + this.unoptimizedQuery = unoptimizedQuery; + this.optimizedQueryString = optimizedQueryString; + this.unoptimizedQueryString = unoptimizedQueryString; + } + + @Override + protected Integer evaluateOriginal(G globalState) { + return optimizedQuery.apply(globalState); + } + + @Override + protected Integer evaluateTransformed(G globalState) { + return unoptimizedQuery.apply(globalState); + } + + @Override + protected boolean sidesDiffer(Integer optimizedCount, Integer unoptimizedCount, G globalState) { + if (optimizedCount == -1 || unoptimizedCount == -1) { + return false; + } + return optimizedCount.intValue() != unoptimizedCount.intValue(); + } + + @Override + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the row counts of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { + renderQueryLines(sb, optimizedQueryString, unoptimizedQueryString); + } + } + + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. + private static void renderQueryLines(StringBuilder sb, String optimizedQueryString, String unoptimizedQueryString) { + sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); + sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs both queries and checks the same error fires. + private static > UnexpectedErrorReproducer errorReproducer( + Function optimizedQuery, Function unoptimizedQuery, String optimizedQueryString, + String unoptimizedQueryString, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + optimizedQuery.apply(globalState); + unoptimizedQuery.apply(globalState); + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, optimizedQueryString, unoptimizedQueryString); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + this.reproducer = null; + } + + @Override + public void check() throws SQLException { + reproducer = null; + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + Z select = gen.generateSelect(); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + + E randomWhereCondition = gen.generateBooleanExpression(); + + boolean shouldUseAggregate = Randomly.getBoolean(); + String optimizedQueryString = gen.generateOptimizedQueryString(select, randomWhereCondition, + shouldUseAggregate); + lastQueryString = optimizedQueryString; + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(optimizedQueryString); + } + + String unoptimizedQueryString = gen.generateUnoptimizedQueryString(select, randomWhereCondition); + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(unoptimizedQueryString); + } + + Function optimizedQuery = state -> shouldUseAggregate + ? extractCounts(optimizedQueryString, errors, state) : countRows(optimizedQueryString, errors, state); + Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); + + int optimizedCount; + int unoptimizedCount; + try { + optimizedCount = optimizedQuery.apply(state); + unoptimizedCount = unoptimizedQuery.apply(state); + } catch (AssertionError unexpectedError) { + reproducer = errorReproducer(optimizedQuery, unoptimizedQuery, optimizedQueryString, unoptimizedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + if (optimizedCount == -1 || unoptimizedCount == -1) { + throw new IgnoreMeException(); + } + + if (unoptimizedCount != optimizedCount) { + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, + unoptimizedQueryString); + + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, + unoptimizedCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", optimizedCount, + unoptimizedCount, firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + @Override + public String getLastQueryString() { + return lastQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); + + int count = 0; + try (SQLancerResultSet rs = q.executeAndGet(state)) { + if (rs == null) { + return -1; + } else { + try { + while (rs.next()) { + count++; + } + } catch (SQLException e) { + count = -1; + } + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw (IgnoreMeException) e; + } + throw new AssertionError(q.getQueryString(), e); + } + return count; + } + + private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState state) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); + int count = 0; + try (SQLancerResultSet rs = q.executeAndGet(state)) { + if (rs == null) { + return -1; + } else { + try { + while (rs.next()) { + count += rs.getInt(1); + } + } catch (SQLException e) { + count = -1; + } + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw (IgnoreMeException) e; + } + throw new AssertionError(q.getQueryString(), e); + } + return count; + } + +} diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java new file mode 100644 index 000000000..cde8a2e4b --- /dev/null +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -0,0 +1,138 @@ +package sqlancer.common.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; +import sqlancer.SQLancerDBConnection; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractRowValue; + +public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E, C extends SQLancerDBConnection> + implements TestOracle { + + protected final ExpectedErrors errors = new ExpectedErrors(); + + /** + * The predicates used in WHERE and JOIN clauses, which yield TRUE for the pivot row. + */ + protected final List rectifiedPredicates = new ArrayList<>(); + + /** + * The generalization of a pivot row, as explained in the "Checking arbitrary expressions" paragraph of the PQS + * paper. + */ + protected List pivotRowExpression = new ArrayList<>(); + protected final S globalState; + protected R pivotRow; + + protected PivotedQuerySynthesisBase(S globalState) { + this.globalState = globalState; + } + + @Override + public final void check() throws Exception { + rectifiedPredicates.clear(); + Query pivotRowQuery = getRectifiedQuery(); + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(pivotRowQuery.getQueryString()); + } + Query isContainedQuery = getContainmentCheckQuery(pivotRowQuery); + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(isContainedQuery.getQueryString()); + } + globalState.getState().getLocalState().log(isContainedQuery.getQueryString()); + // combines step 6 and 7 described in the PQS paper + boolean pivotRowIsContained = containsRows(isContainedQuery); + if (!pivotRowIsContained) { + reportMissingPivotRow(pivotRowQuery); + } + } + + /** + * Checks whether the result set contains at least a single row. + * + * @param query + * the query for which to check whether its result set contains any rows + * + * @return true if at least one row is contained, false otherwise + * + * @throws Exception + * if the query unexpectedly fails + */ + private boolean containsRows(Query query) throws Exception { + try (SQLancerResultSet result = query.executeAndGet(globalState)) { + if (result == null) { + throw new IgnoreMeException(); + } + return !result.isClosed(); + } + } + + protected void reportMissingPivotRow(Query query) { + globalState.getState().getLocalState().log("-- pivot row values:"); + String expectedPivotRowString = pivotRow.asStringGroupedByTables(); + globalState.getState().getLocalState().log(expectedPivotRowString); + + StringBuilder sb = new StringBuilder(); + if (!rectifiedPredicates.isEmpty()) { + sb.append("--\n-- rectified predicates and their expected values:\n"); + for (E rectifiedPredicate : rectifiedPredicates) { + sb.append("--"); + sb.append(getExpectedValues(rectifiedPredicate).replace("\n", "\n-- ")); + } + sb.append("\n"); + } + if (!pivotRowExpression.isEmpty()) { + sb.append("-- pivot row expressions and their expected values:\n"); + for (E pivotRowExpression : pivotRowExpression) { + sb.append("--"); + sb.append(getExpectedValues(pivotRowExpression).replace("\n", "\n--")); + sb.append("\n"); + } + } + globalState.getState().getLocalState().log(sb.toString()); + throw new AssertionError(query); + } + + /** + * Gets a query that checks whether the pivot row is contained in the result. If the pivot row is contained, the + * query will fetch at least one row. If the pivot row is not contained, no rows will be fetched. This corresponds + * to step 7 described in the PQS paper. + * + * @param pivotRowQuery + * the query that is guaranteed to fetch the pivot row, potentially among other rows + * + * @return a query that checks whether the pivot row is contained in pivotRowQuery + * + * @throws Exception + * if an unexpected error occurs + */ + protected abstract Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception; + + /** + * Obtains a rectified query (i.e., a query that is guaranteed to fetch the pivot row. This corresponds to steps 2-5 + * of the PQS paper. + * + * @return the rectified query + * + * @throws Exception + * if an unexpected error occurs + */ + protected abstract Query getRectifiedQuery() throws Exception; + + /** + * Prints the value to which the expression is expected to evaluate, and then recursively prints the subexpressions' + * expected values. + * + * @param expr + * the expression whose expected value should be printed + * + * @return a string representing the expected value of the expression and its subexpressions + */ + protected abstract String getExpectedValues(E expr); + +} diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java new file mode 100644 index 000000000..5e1d2861b --- /dev/null +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -0,0 +1,206 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class TLPWhereOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + + private TLPWhereGenerator gen; + private final ExpectedErrors errors; + + private Reproducer reproducer; + private String generatedQueryString; + + // A side's result set, together with the human-readable combined-query strings that + // getCombinedResultSet fills in for the transformed side (unused, and null, for the original side) + private static final class TLPResultSet { + final List resultSet; + final List combinedString; + + TLPResultSet(List resultSet, List combinedString) { + this.resultSet = resultSet; + this.combinedString = combinedString; + } + } + + private class TLPWhereReproducer extends AbstractComparisonReproducer { + final String firstQueryString; + final String secondQueryString; + final String thirdQueryString; + final String originalQueryString; + final boolean orderBy; + + TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, + String originalQueryString, boolean orderBy) { + this.firstQueryString = firstQueryString; + this.secondQueryString = secondQueryString; + this.thirdQueryString = thirdQueryString; + this.originalQueryString = originalQueryString; + this.orderBy = orderBy; + } + + @Override + protected TLPResultSet evaluateOriginal(G globalState) throws SQLException { + return new TLPResultSet( + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState), null); + } + + @Override + protected TLPResultSet evaluateTransformed(G globalState) throws SQLException { + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, globalState, errors); + return new TLPResultSet(secondResultSet, combinedString); + } + + @Override + protected boolean sidesDiffer(TLPResultSet original, TLPResultSet transformed, G globalState) { + try { + ComparatorHelper.assumeResultSetsAreEqual(original.resultSet, transformed.resultSet, + originalQueryString, transformed.combinedString, globalState); + } catch (AssertionError resultSetMismatch) { + return true; + } + return false; + } + + @Override + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { + renderQueryLines(sb, originalQueryString, firstQueryString, secondQueryString, thirdQueryString, orderBy); + } + } + + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. The + // partition queries are null when the error struck the original query before they were built. + private static void renderQueryLines(StringBuilder sb, String originalQueryString, String firstQueryString, + String secondQueryString, String thirdQueryString, boolean orderBy) { + sb.append("-- ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (firstQueryString != null) { + if (orderBy) { + sb.append("-- ").append(firstQueryString).append(';').append(System.lineSeparator()); + sb.append("-- ").append(secondQueryString).append(';').append(System.lineSeparator()); + sb.append("-- ").append(thirdQueryString).append(';').append(System.lineSeparator()); + } else { + sb.append("-- ").append(firstQueryString).append(" UNION ALL ").append(secondQueryString) + .append(" UNION ALL ").append(thirdQueryString).append(';').append(System.lineSeparator()); + } + } + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs the query (or the whole-table query plus the + // three partition queries) and checks the same error still fires. The partition queries are null when the error + // struck the original query before they were built. + private UnexpectedErrorReproducer errorReproducer(String originalQueryString, String firstQueryString, + String secondQueryString, String thirdQueryString, boolean orderBy, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + if (firstQueryString != null) { + ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, + new ArrayList<>(), !orderBy, globalState, errors); + } + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, originalQueryString, firstQueryString, secondQueryString, thirdQueryString, orderBy); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + reproducer = null; + S s = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(s); + gen = gen.setTablesAndColumns(targetTables); + + Select select = gen.generateSelect(); + + boolean shouldCreateDummy = true; + select.setFetchColumns(gen.generateFetchColumns(shouldCreateDummy)); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + select.setWhereClause(null); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List firstResultSet; + try { + firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + reproducer = errorReproducer(originalQueryString, null, null, null, false, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + boolean orderBy = Randomly.getBooleanWithSmallProbability(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + + TestOracleUtils.PredicateVariants predicates = TestOracleUtils.initializeTernaryPredicateVariants(gen, + gen.generateBooleanExpression()); + select.setWhereClause(predicates.predicate); + String firstQueryString = select.asString(); + select.setWhereClause(predicates.negatedPredicate); + String secondQueryString = select.asString(); + select.setWhereClause(predicates.isNullPredicate); + String thirdQueryString = select.asString(); + + List combinedString = new ArrayList<>(); + List secondResultSet; + try { + secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + } catch (AssertionError unexpectedError) { + reproducer = errorReproducer(originalQueryString, firstQueryString, secondQueryString, thirdQueryString, + orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + orderBy); + ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java new file mode 100644 index 000000000..b4d6add4b --- /dev/null +++ b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java @@ -0,0 +1,55 @@ +package sqlancer.common.oracle; + +import sqlancer.GlobalState; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.query.ExpectedErrors; + +/** + * This is the base class of the Ternary Logic Partitioning (TLP) oracles. The core idea of TLP is to partition a given + * so-called original query to three so-called partitioning queries, each of which computes a partition of the original + * query's result. + * + * @param + * the expression type + * @param + * the global state type + */ +public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { + + protected E predicate; + protected E negatedPredicate; + protected E isNullPredicate; + + protected final S state; + protected final ExpectedErrors errors = new ExpectedErrors(); + + protected TernaryLogicPartitioningOracleBase(S state) { + this.state = state; + } + + protected E generatePredicate() { + return getGen().generatePredicate(); + } + + protected void initializeTernaryPredicateVariants() { + ExpressionGenerator gen = getGen(); + if (gen == null) { + throw new IllegalStateException(); + } + predicate = generatePredicate(); + if (predicate == null) { + throw new IllegalStateException(); + } + negatedPredicate = gen.negatePredicate(predicate); + if (negatedPredicate == null) { + throw new IllegalStateException(); + } + isNullPredicate = gen.isNull(predicate); + if (isNullPredicate == null) { + throw new IllegalStateException(); + } + } + + protected abstract ExpressionGenerator getGen(); + +} diff --git a/src/sqlancer/common/oracle/TestOracle.java b/src/sqlancer/common/oracle/TestOracle.java new file mode 100644 index 000000000..0ef993b89 --- /dev/null +++ b/src/sqlancer/common/oracle/TestOracle.java @@ -0,0 +1,17 @@ +package sqlancer.common.oracle; + +import sqlancer.GlobalState; +import sqlancer.Reproducer; + +public interface TestOracle> { + + void check() throws Exception; + + default Reproducer getLastReproducer() { + return null; + } + + default String getLastQueryString() { + throw new AssertionError("Not supported!"); + } +} diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java new file mode 100644 index 000000000..9bef86762 --- /dev/null +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -0,0 +1,77 @@ +package sqlancer.common.oracle; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.gen.PartitionGenerator; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public final class TestOracleUtils { + + private TestOracleUtils() { + } + + public static final class PredicateVariants, C extends AbstractTableColumn> { + public E predicate; + public E negatedPredicate; + public E isNullPredicate; + + PredicateVariants(E predicate, E negatedPredicate, E isNullPredicate) { + this.predicate = predicate; + this.negatedPredicate = negatedPredicate; + this.isNullPredicate = isNullPredicate; + } + } + + public static , C extends AbstractTableColumn> AbstractTables getRandomTableNonEmptyTables( + AbstractSchema schema) { + if (schema.getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); + } + + /** + * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the AssertionError + * that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it to check that a + * reduced test case still triggers the same error, rather than an unrelated one introduced by the reduction itself. + * + * @param error + * the AssertionError wrapping the DBMS error + * + * @return the message of the innermost cause that has one, or the error's own message + */ + public static String getUnexpectedErrorMessage(AssertionError error) { + String message = error.getMessage(); + Throwable current = error.getCause(); + while (current != null) { + if (current.getMessage() != null) { + message = current.getMessage(); + } + current = current.getCause(); + } + return message; + } + + public static , T extends AbstractTable, C extends AbstractTableColumn> PredicateVariants initializeTernaryPredicateVariants( + PartitionGenerator gen, E predicate) { + if (gen == null) { + throw new IllegalStateException(); + } + if (predicate == null) { + throw new IllegalStateException(); + } + E negatedPredicate = gen.negatePredicate(predicate); + if (negatedPredicate == null) { + throw new IllegalStateException(); + } + E isNullPredicate = gen.isNull(predicate); + if (isNullPredicate == null) { + throw new IllegalStateException(); + } + return new PredicateVariants<>(predicate, negatedPredicate, isNullPredicate); + } +} diff --git a/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java b/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java new file mode 100644 index 000000000..e9390943a --- /dev/null +++ b/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java @@ -0,0 +1,79 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; + +/** + * Reproducer for a bug that is an unexpected DBMS error, rather than a violation of oracle logic. When a statement run + * by the oracle raises an error the oracle did not expect, the bug is that error. Reduction re-runs the statement + * execution against the reduced database and reports whether the same error still fires. The oracle supplies how to + * re-run its execution as an {@link Execution} functional interface. + * + *

+ * For an oracle to use this reproducer on its unexpected errors, they must surface as {@link AssertionError}s, even + * though they likely originated as {@link SQLException}s. This is because + * {@link UnexpectedErrorReproducer#bugStillTriggers} treats a {@link SQLException} as a replay failure that means "bug + * does not trigger" (e.g. dropped connection, removed table). + * + * @param + * the DBMS-specific global state class + */ +public final class UnexpectedErrorReproducer> implements Reproducer { + + private final Execution execution; + private final String expectedErrorMessage; + private final String queryLines; + + @FunctionalInterface + public interface Execution { + /** + * Re-runs the oracle's execution against the reduced database. The bug is treated as still present if the run + * continues to raise an {@link AssertionError} with the same message. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @throws SQLException + * if a DBMS interaction fails for a reason other than the recorded bug (e.g. a connection or setup + * failure during replay), which counts as the bug no longer triggering + */ + void execute(G globalState) throws SQLException; + } + + /** + * @param execution + * re-runs the oracle's execution against the reduced database + * @param expectedErrorMessage + * the message of the error the original bug was. The bug is treated as still present if the + * {@link Execution} continues to raise an {@link AssertionError} with the same message. + * @param queryLines + * the failing queries as commented lines (each ending in a line separator), for the reduced test case + */ + public UnexpectedErrorReproducer(Execution execution, String expectedErrorMessage, String queryLines) { + this.execution = execution; + this.expectedErrorMessage = expectedErrorMessage; + this.queryLines = queryLines; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + execution.execute(globalState); + } catch (AssertionError unexpectedError) { + // the same error reproduces the bug; a different one is an artifact of the reduction (e.g. a removed table) + return expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + // the error no longer fires + return false; + } + + @Override + public String getBugInformation() { + return "-- On the database set up by the statements above, the following queries trigger an unexpected error" + + " with message: " + expectedErrorMessage + System.lineSeparator() + queryLines; + } +} diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java new file mode 100644 index 000000000..eb281efbe --- /dev/null +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -0,0 +1,149 @@ +package sqlancer.common.query; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * This class represents the errors that executing a statement might result in. For example, an INSERT statement might + * result in an error "UNIQUE constraint violated" when it attempts to insert a duplicate value in a column declared as + * UNIQUE. + */ +public class ExpectedErrors implements Serializable { + private static final long serialVersionUID = 1L; + + private final Set errors; + private final Set regexes; + + public ExpectedErrors() { + this.errors = new HashSet<>(); + this.regexes = new HashSet<>(); + } + + public ExpectedErrors(Collection errors, Collection regexErrors) { + this.errors = new HashSet<>(errors); + this.regexes = new HashSet<>(regexErrors); + } + + public ExpectedErrors add(String error) { + if (error == null) { + throw new IllegalArgumentException(); + } + errors.add(error); + return this; + } + + public ExpectedErrors addRegex(Pattern errorPattern) { + if (errorPattern == null) { + throw new IllegalArgumentException(); + } + regexes.add(errorPattern); + return this; + } + + public ExpectedErrors addRegexString(String errorPattern) { + if (errorPattern == null) { + throw new IllegalArgumentException(); + } + regexes.add(Pattern.compile(errorPattern)); + return this; + } + + public ExpectedErrors addAll(Collection list) { + if (list == null) { + throw new IllegalArgumentException(); + } + errors.addAll(list); + return this; + } + + public ExpectedErrors addAllRegexes(Collection list) { + if (list == null) { + throw new IllegalArgumentException(); + } + regexes.addAll(list); + return this; + } + + public ExpectedErrors addAllRegexStrings(Collection list) { + for (String error : list) { + regexes.add(Pattern.compile(error)); + } + return this; + } + + public static ExpectedErrors from(String... errors) { + return newErrors().with(errors).build(); + } + + public static ExpectedErrorsBuilder newErrors() { + return new ExpectedErrorsBuilder(); + } + + /** + * Checks whether the error message (e.g., returned by the DBMS under test) contains any of the added error + * messages. + * + * @param error + * the error message + * + * @return whether the error message contains any of the substrings specified as expected errors + */ + public boolean errorIsExpected(String error) { + if (error == null) { + throw new IllegalArgumentException(); + } + for (String s : this.errors) { + if (error.contains(s)) { + return true; + } + } + for (Pattern p : this.regexes) { + if (p.matcher(error).find()) { + return true; + } + } + return false; + } + + public static class ExpectedErrorsBuilder { + private final Set errors = new HashSet<>(); + private final Set regexes = new HashSet<>(); + + public ExpectedErrorsBuilder with(String... list) { + errors.addAll(Arrays.asList(list)); + return this; + } + + public ExpectedErrorsBuilder with(Collection list) { + return with(list.toArray(new String[0])); + } + + public ExpectedErrorsBuilder withRegex(Pattern... list) { + regexes.addAll(Arrays.asList(list)); + return this; + } + + public ExpectedErrorsBuilder withRegex(Collection list) { + return withRegex(list.toArray(new Pattern[0])); + } + + public ExpectedErrorsBuilder withRegexString(String... list) { + for (String error : list) { + regexes.add(Pattern.compile(error)); + } + return this; + } + + public ExpectedErrorsBuilder withRegexString(Collection list) { + return withRegexString(list.toArray(new String[0])); + } + + public ExpectedErrors build() { + return new ExpectedErrors(errors, regexes); + } + } +} diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java new file mode 100644 index 000000000..ca90619c7 --- /dev/null +++ b/src/sqlancer/common/query/Query.java @@ -0,0 +1,61 @@ +package sqlancer.common.query; + +import sqlancer.GlobalState; +import sqlancer.SQLancerDBConnection; +import sqlancer.common.log.Loggable; + +public abstract class Query implements Loggable { + private static final long serialVersionUID = 1L; + + /** + * Gets the query string, which is guaranteed to be terminated with a semicolon. + * + * @return the query string. + */ + public abstract String getQueryString(); + + /** + * Gets the query string without trailing semicolons. + * + * @return the query string that does not end with a ";". + */ + public abstract String getUnterminatedQueryString(); + + /** + * Whether the query could affect the schema (i.e., by add/deleting columns or tables). + * + * @return true if the query can affect the database's schema, false otherwise + */ + public abstract boolean couldAffectSchema(); + + public abstract > boolean execute(G globalState, String... fills) throws Exception; + + public abstract ExpectedErrors getExpectedErrors(); + + @Override + public String toString() { + return getQueryString(); + } + + public > SQLancerResultSet executeAndGet(G globalState, String... fills) + throws Exception { + throw new AssertionError(); + } + + public > boolean executeLogged(G globalState) throws Exception { + logQueryString(globalState); + return execute(globalState); + } + + public > SQLancerResultSet executeAndGetLogged(G globalState) throws Exception { + logQueryString(globalState); + return executeAndGet(globalState); + } + + private > void logQueryString(G globalState) { + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(getQueryString()); + } + } + +} diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java new file mode 100644 index 000000000..db8a2c66d --- /dev/null +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -0,0 +1,227 @@ +package sqlancer.common.query; + +import java.io.Serializable; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import sqlancer.GlobalState; +import sqlancer.Main; +import sqlancer.SQLConnection; + +public class SQLQueryAdapter extends Query implements Serializable { + private static final long serialVersionUID = 1L; + + private final String query; + private final ExpectedErrors expectedErrors; + private final boolean couldAffectSchema; + + public SQLQueryAdapter(String query) { + this(query, new ExpectedErrors()); + } + + public SQLQueryAdapter(String query, boolean couldAffectSchema) { + this(query, new ExpectedErrors(), couldAffectSchema); + } + + public SQLQueryAdapter(String query, ExpectedErrors expectedErrors) { + this(query, expectedErrors, guessAffectSchemaFromQuery(query)); + } + + private static boolean guessAffectSchemaFromQuery(String query) { + return query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN"); + } + + public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema) { + this(query, expectedErrors, couldAffectSchema, true); + } + + public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema, + boolean canonicalizeString) { + if (canonicalizeString) { + this.query = canonicalizeString(query); + } else { + this.query = query; + } + this.expectedErrors = expectedErrors; + this.couldAffectSchema = couldAffectSchema; + checkQueryString(); + } + + private String canonicalizeString(String s) { + if (s.endsWith(";")) { + return s; + } else if (!s.contains("--")) { + return s + ";"; + } else { + // query contains a comment + return s; + } + } + + private void checkQueryString() { + if (!couldAffectSchema && guessAffectSchemaFromQuery(query)) { + throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); + } + } + + @Override + public String getQueryString() { + return query; + } + + @Override + public String getUnterminatedQueryString() { + String result; + if (query.endsWith(";")) { + result = query.substring(0, query.length() - 1); + } else { + result = query; + } + assert !result.endsWith(";"); + return result; + } + + /** + * This method is used to mostly oracles, which need to report exceptions. We set the reportException parameter to + * true by default meaning that exceptions are reported. + * + * @param globalState + * @param fills + * + * @return whether the query was executed successfully + * + * @param + * + * @throws SQLException + */ + @Override + public > boolean execute(G globalState, String... fills) + throws SQLException { + return execute(globalState, true, fills); + } + + /** + * This method is used to DQE oracles, DQE does not check exception separately, while other testing methods may + * need. We use reportException to control this behavior. For a specific DBMS used DQE oracle, we call this method + * and pass a boolean value of false as an argument. + * + * @param globalState + * @param reportException + * @param fills + * + * @return whether the query was executed successfully + * + * @param + * + * @throws SQLException + */ + public > boolean execute(G globalState, boolean reportException, + String... fills) throws SQLException { + return internalExecute(globalState.getConnection(), reportException, fills); + } + + protected > boolean internalExecute(SQLConnection connection, + boolean reportException, String... fills) throws SQLException { + Statement s; + if (fills.length > 0) { + s = connection.prepareStatement(fills[0]); + for (int i = 1; i < fills.length; i++) { + ((PreparedStatement) s).setString(i, fills[i]); + } + } else { + s = connection.createStatement(); + } + try { + if (fills.length > 0) { + ((PreparedStatement) s).execute(); + } else { + s.execute(query); + } + Main.nrSuccessfulActions.addAndGet(1); + return true; + } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (reportException) { + checkException(e); + } + return false; + } finally { + s.close(); + } + } + + public void checkException(Exception e) throws AssertionError { + Throwable ex = e; + + while (ex != null) { + if (expectedErrors.errorIsExpected(ex.getMessage())) { + return; + } else { + ex = ex.getCause(); + } + } + + throw new AssertionError(query, e); + } + + @Override + public > SQLancerResultSet executeAndGet(G globalState, String... fills) + throws SQLException { + return executeAndGet(globalState, true, fills); + } + + public > SQLancerResultSet executeAndGet(G globalState, + boolean reportException, String... fills) throws SQLException { + return internalExecuteAndGet(globalState.getConnection(), reportException, fills); + } + + protected > SQLancerResultSet internalExecuteAndGet( + SQLConnection connection, boolean reportException, String... fills) throws SQLException { + Statement s; + if (fills.length > 0) { + s = connection.prepareStatement(fills[0]); + for (int i = 1; i < fills.length; i++) { + ((PreparedStatement) s).setString(i, fills[i]); + } + } else { + s = connection.createStatement(); + } + ResultSet result; + try { + if (fills.length > 0) { + result = ((PreparedStatement) s).executeQuery(); + } else { + result = s.executeQuery(query); + } + Main.nrSuccessfulActions.addAndGet(1); + if (result == null) { + return null; + } + return new SQLancerResultSet(result); + } catch (Exception e) { + s.close(); + Main.nrUnsuccessfulActions.addAndGet(1); + if (reportException) { + checkException(e); + } + return null; + } + } + + @Override + public boolean couldAffectSchema() { + return couldAffectSchema; + } + + @Override + public ExpectedErrors getExpectedErrors() { + return expectedErrors; + } + + @Override + public String getLogString() { + return getQueryString(); + } +} diff --git a/src/sqlancer/common/query/SQLQueryError.java b/src/sqlancer/common/query/SQLQueryError.java new file mode 100644 index 000000000..41f604930 --- /dev/null +++ b/src/sqlancer/common/query/SQLQueryError.java @@ -0,0 +1,116 @@ +package sqlancer.common.query; + +import java.util.Objects; + +public class SQLQueryError implements Comparable { + + public enum ErrorLevel { + WARNING, ERROR + } + + private ErrorLevel level; + private int code; + private String message; + + public void setLevel(ErrorLevel level) { + this.level = level; + } + + public void setCode(int code) { + this.code = code; + } + + public void setMessage(String message) { + this.message = message; + } + + public ErrorLevel getLevel() { + return level; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public boolean hasSameLevel(SQLQueryError that) { + if (level == null) { + return that.getLevel() == null; + } else { + return level.equals(that.getLevel()); + } + } + + public boolean hasSameCodeAndMessage(SQLQueryError that) { + if (code != that.getCode()) { + return false; + } + if (message == null) { + return that.getMessage() == null; + } else { + return message.equals(that.getMessage()); + } + } + + @Override + public boolean equals(Object that) { + if (that == null) { + return false; + } + if (that instanceof SQLQueryError) { + SQLQueryError thatError = (SQLQueryError) that; + return hasSameLevel(thatError) && hasSameCodeAndMessage(thatError); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(level, code, message); + } + + @Override + public String toString() { + return String.format("Level: %s; Code: %d; Message: %s.", level, code, message); + } + + @Override + public int compareTo(SQLQueryError that) { + if (code < that.getCode()) { + return -1; + } else if (code > that.getCode()) { + return 1; + } + + if (level == null && that.getLevel() != null) { + return -1; + } else { + if (that.getLevel() == null) { + return 1; + } else { + int res = level.compareTo(that.getLevel()); + if (res != 0) { + return res; + } + } + } + + if (message == null && that.getMessage() != null) { + return -1; + } else { + if (that.getMessage() == null) { + return 1; + } else { + int res = message.compareTo(that.getMessage()); + if (res != 0) { + return res; + } + } + } + + return 0; + } +} diff --git a/src/sqlancer/common/query/SQLQueryProvider.java b/src/sqlancer/common/query/SQLQueryProvider.java new file mode 100644 index 000000000..39aa0e05c --- /dev/null +++ b/src/sqlancer/common/query/SQLQueryProvider.java @@ -0,0 +1,6 @@ +package sqlancer.common.query; + +@FunctionalInterface +public interface SQLQueryProvider { + SQLQueryAdapter getQuery(S globalState) throws Exception; +} diff --git a/src/sqlancer/QueryResultCheckAdapter.java b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java similarity index 56% rename from src/sqlancer/QueryResultCheckAdapter.java rename to src/sqlancer/common/query/SQLQueryResultCheckAdapter.java index c8f294a02..1eaae2424 100644 --- a/src/sqlancer/QueryResultCheckAdapter.java +++ b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java @@ -1,21 +1,26 @@ -package sqlancer; +package sqlancer.common.query; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.function.Consumer; -public class QueryResultCheckAdapter extends QueryAdapter { +import sqlancer.GlobalState; +import sqlancer.SQLConnection; + +public class SQLQueryResultCheckAdapter extends SQLQueryAdapter { + private static final long serialVersionUID = 1L; private final Consumer rsChecker; - public QueryResultCheckAdapter(String query, Consumer rsChecker) { + public SQLQueryResultCheckAdapter(String query, Consumer rsChecker) { super(query); this.rsChecker = rsChecker; } @Override - public boolean execute(GlobalState globalState) throws SQLException { + public > boolean execute(G globalState, String... fills) + throws SQLException { try (Statement s = globalState.getConnection().createStatement()) { ResultSet rs = s.executeQuery(getQueryString()); rsChecker.accept(rs); diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java new file mode 100644 index 000000000..912a6a882 --- /dev/null +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -0,0 +1,69 @@ +package sqlancer.common.query; + +import java.io.Closeable; +import java.sql.ResultSet; +import java.sql.SQLException; + +public class SQLancerResultSet implements Closeable { + + ResultSet rs; + private Runnable runnableEpilogue; + + public SQLancerResultSet(ResultSet rs) { + this.rs = rs; + } + + @Override + public void close() { + try { + if (runnableEpilogue != null) { + runnableEpilogue.run(); + } + rs.getStatement().close(); + rs.close(); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + public boolean next() throws SQLException { + return rs.next(); + } + + public int getInt(int i) throws SQLException { + return rs.getInt(i); + } + + public String getString(int i) throws SQLException { + try { + return rs.getString(i); + } catch (NumberFormatException e) { + throw new SQLException(e); + } + } + + public String getString(String colName) throws SQLException { + return rs.getString(colName); + } + + public int getInt(String colName) throws SQLException { + return rs.getInt(colName); + } + + public boolean isClosed() throws SQLException { + return rs.isClosed(); + } + + public long getLong(int i) throws SQLException { + return rs.getLong(i); + } + + public String getType(int i) throws SQLException { + return rs.getMetaData().getColumnTypeName(i); + } + + public void registerEpilogue(Runnable runnableEpilogue) { + this.runnableEpilogue = runnableEpilogue; + } + +} diff --git a/src/sqlancer/common/schema/AbstractRelationalTable.java b/src/sqlancer/common/schema/AbstractRelationalTable.java new file mode 100644 index 000000000..7f26efc1a --- /dev/null +++ b/src/sqlancer/common/schema/AbstractRelationalTable.java @@ -0,0 +1,37 @@ +package sqlancer.common.schema; + +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + +public class AbstractRelationalTable, I extends TableIndex, G extends SQLGlobalState> + extends AbstractTable { + + public AbstractRelationalTable(String name, List columns, List indexes, boolean isView) { + super(name, columns, indexes, isView); + } + + @Override + public long getNrRows(G globalState) { + if (rowCount == NO_ROW_COUNT_AVAILABLE) { + SQLQueryAdapter q = new SQLQueryAdapter("SELECT COUNT(*) FROM " + name); + try (SQLancerResultSet query = q.executeAndGet(globalState)) { + if (query == null) { + throw new IgnoreMeException(); + } + query.next(); + rowCount = query.getLong(1); + return rowCount; + } catch (Throwable t) { + // an exception might be expected, for example, when invalid view is created + throw new IgnoreMeException(); + } + } else { + return rowCount; + } + } + +} diff --git a/src/sqlancer/common/schema/AbstractRowValue.java b/src/sqlancer/common/schema/AbstractRowValue.java new file mode 100644 index 000000000..2a979514d --- /dev/null +++ b/src/sqlancer/common/schema/AbstractRowValue.java @@ -0,0 +1,82 @@ +package sqlancer.common.schema; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public abstract class AbstractRowValue, C extends AbstractTableColumn, O> { + + private final T tables; + private final Map values; + + protected AbstractRowValue(T tables, Map values) { + this.tables = tables; + this.values = values; + } + + public T getTable() { + return tables; + } + + public Map getValues() { + return values; + } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer(); + int i = 0; + for (C c : tables.getColumns()) { + if (i++ != 0) { + sb.append(", "); + } + sb.append(values.get(c)); + } + return sb.toString(); + } + + public String getRowValuesAsString() { + List columnsToCheck = tables.getColumns(); + return getRowValuesAsString(columnsToCheck); + } + + public String getRowValuesAsString(List columnsToCheck) { + StringBuilder sb = new StringBuilder(); + Map expectedValues = getValues(); + for (int i = 0; i < columnsToCheck.size(); i++) { + if (i != 0) { + sb.append(", "); + } + O expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); + sb.append(expectedColumnValue); + } + return sb.toString(); + } + + public String asStringGroupedByTables() { + StringBuilder sb = new StringBuilder(); + List columnList = getValues().keySet().stream().collect(Collectors.toList()); + List> tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() + .collect(Collectors.toList()); + for (int j = 0; j < tableList.size(); j++) { + if (j != 0) { + sb.append("\n"); + } + AbstractTable t = tableList.get(j); + sb.append("-- ").append(t.getName()).append("\n"); + List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) + .collect(Collectors.toList()); + for (int i = 0; i < columnsForTable.size(); i++) { + if (i != 0) { + sb.append("\n"); + } + sb.append("--\t"); + sb.append(columnsForTable.get(i)); + sb.append("="); + sb.append(getValues().get(columnsForTable.get(i))); + } + } + return sb.toString(); + } + +} diff --git a/src/sqlancer/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java similarity index 53% rename from src/sqlancer/schema/AbstractSchema.java rename to src/sqlancer/common/schema/AbstractSchema.java index bb8014a54..053c9540a 100644 --- a/src/sqlancer/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -1,13 +1,16 @@ -package sqlancer.schema; +package sqlancer.common.schema; import java.util.Collections; import java.util.List; +import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; -public class AbstractSchema> { +public class AbstractSchema, A extends AbstractTable> { private final List databaseTables; @@ -19,7 +22,8 @@ public AbstractSchema(List databaseTables) { public String toString() { StringBuffer sb = new StringBuffer(); for (A t : getDatabaseTables()) { - sb.append(t + "\n"); + sb.append(t); + sb.append("\n"); } return sb.toString(); } @@ -28,18 +32,66 @@ public A getRandomTable() { return Randomly.fromList(getDatabaseTables()); } + public A getRandomTableOrBailout() { + if (databaseTables.isEmpty()) { + throw new IgnoreMeException(); + } else { + return Randomly.fromList(getDatabaseTables()); + } + } + public A getRandomTable(Predicate predicate) { return Randomly.fromList(getDatabaseTables().stream().filter(predicate).collect(Collectors.toList())); } + public A getRandomTableOrBailout(Function f) { + List relevantTables = databaseTables.stream().filter(f::apply).collect(Collectors.toList()); + if (relevantTables.isEmpty()) { + throw new IgnoreMeException(); + } + return Randomly.fromList(relevantTables); + } + public List getDatabaseTables() { return databaseTables; } + public List getTables(Predicate predicate) { + return databaseTables.stream().filter(predicate).collect(Collectors.toList()); + } + public List getDatabaseTablesRandomSubsetNotEmpty() { return Randomly.nonEmptySubset(databaseTables); } + public A getDatabaseTable(String name) { + return databaseTables.stream().filter(t -> t.getName().equals(name)).findAny().orElse(null); + } + + public List getViews() { + return databaseTables.stream().filter(t -> t.isView()).collect(Collectors.toList()); + } + + public List getDatabaseTablesWithoutViews() { + return databaseTables.stream().filter(t -> !t.isView()).collect(Collectors.toList()); + } + + public A getRandomViewOrBailout() { + if (getViews().isEmpty()) { + throw new IgnoreMeException(); + } else { + return Randomly.fromList(getViews()); + } + } + + public A getRandomTableNoViewOrBailout() { + List databaseTablesWithoutViews = getDatabaseTablesWithoutViews(); + if (databaseTablesWithoutViews.isEmpty()) { + throw new IgnoreMeException(); + } + return Randomly.fromList(databaseTablesWithoutViews); + } + public String getFreeIndexName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { @@ -67,13 +119,17 @@ public String getFreeTableName() { } do { String tableName = String.format("t%d", i++); - if (databaseTables.stream().noneMatch(t -> t.getName().contentEquals(tableName))) { + if (databaseTables.stream().noneMatch(t -> t.getName().equalsIgnoreCase(tableName))) { return tableName; } } while (true); } + public static boolean matchesViewName(String relationName) { + return relationName.startsWith("v"); + } + public String getFreeViewName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { @@ -85,7 +141,10 @@ public String getFreeViewName() { return tableName; } } while (true); + } + public boolean containsTableWithZeroRows(G globalState) { + return databaseTables.stream().anyMatch(t -> t.getNrRows(globalState) == 0); } } diff --git a/src/sqlancer/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java similarity index 56% rename from src/sqlancer/schema/AbstractTable.java rename to src/sqlancer/common/schema/AbstractTable.java index 749d7f878..58154681c 100644 --- a/src/sqlancer/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -1,21 +1,25 @@ -package sqlancer.schema; +package sqlancer.common.schema; import java.util.Collections; import java.util.List; -import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; +import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; -public class AbstractTable, I extends TableIndex> - implements Comparable> { +public abstract class AbstractTable, I extends TableIndex, G extends GlobalState> + implements Comparable> { - private final String name; + protected static final int NO_ROW_COUNT_AVAILABLE = -1; + protected final String name; private final List columns; private final List indexes; private final boolean isView; + protected long rowCount = NO_ROW_COUNT_AVAILABLE; - public AbstractTable(String name, List columns, List indexes, boolean isView) { + protected AbstractTable(String name, List columns, List indexes, boolean isView) { this.name = name; this.indexes = indexes; this.isView = isView; @@ -27,16 +31,17 @@ public String getName() { } @Override - public int compareTo(AbstractTable o) { + public int compareTo(AbstractTable o) { return o.getName().compareTo(getName()); } @Override public String toString() { StringBuffer sb = new StringBuffer(); - sb.append(getName() + "\n"); + sb.append(getName()); + sb.append("\n"); for (C c : columns) { - sb.append("\t" + c + "\n"); + sb.append("\t").append(c).append("\n"); } return sb.toString(); } @@ -53,14 +58,19 @@ public String getColumnsAsString() { return columns.stream().map(c -> c.getName()).collect(Collectors.joining(", ")); } - public String getColumnsAsString(Function function) { - return columns.stream().map(function).collect(Collectors.joining(", ")); - } - public C getRandomColumn() { return Randomly.fromList(columns); } + public C getRandomColumnOrBailout(Predicate predicate) { + List relevantColumns = columns.stream().filter(predicate).collect(Collectors.toList()); + if (relevantColumns.isEmpty()) { + throw new IgnoreMeException(); + } + + return Randomly.fromList(relevantColumns); + } + public boolean hasIndexes() { return !indexes.isEmpty(); } @@ -73,6 +83,10 @@ public List getRandomNonEmptyColumnSubset() { return Randomly.nonEmptySubset(getColumns()); } + public List getRandomNonEmptyColumnSubsetFilter(Predicate predicate) { + return Randomly.nonEmptySubset(getColumns().stream().filter(predicate).collect(Collectors.toList())); + } + public List getRandomNonEmptyColumnSubset(int size) { return Randomly.nonEmptySubset(getColumns(), size); } @@ -81,6 +95,10 @@ public boolean isView() { return isView; } + public boolean hasPrimaryKey() { + return columns.stream().anyMatch(c -> c.isPrimaryKey()); + } + public String getFreeColumnName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { @@ -95,4 +113,9 @@ public String getFreeColumnName() { } + public void recomputeCount() { + rowCount = NO_ROW_COUNT_AVAILABLE; + } + + public abstract long getNrRows(G globalState); } diff --git a/src/sqlancer/schema/AbstractTableColumn.java b/src/sqlancer/common/schema/AbstractTableColumn.java similarity index 78% rename from src/sqlancer/schema/AbstractTableColumn.java rename to src/sqlancer/common/schema/AbstractTableColumn.java index 07c5b2fee..a2f5fb1b3 100644 --- a/src/sqlancer/schema/AbstractTableColumn.java +++ b/src/sqlancer/common/schema/AbstractTableColumn.java @@ -1,6 +1,6 @@ -package sqlancer.schema; +package sqlancer.common.schema; -public class AbstractTableColumn, U> implements Comparable> { +public class AbstractTableColumn, U> implements Comparable> { private final String name; private final U type; @@ -12,6 +12,10 @@ public AbstractTableColumn(String name, T table, U type) { this.type = type; } + public boolean isPrimaryKey() { + return false; + } + public String getName() { return name; } @@ -52,7 +56,10 @@ public boolean equals(Object obj) { } else { @SuppressWarnings("unchecked") AbstractTableColumn c = (AbstractTableColumn) obj; - return table.getName().contentEquals(getName()) && getName().equals(c.getName()); + if (c.getTable() == null) { + return getName().equals(c.getName()); + } + return table.getName().contentEquals(c.getTable().getName()) && getName().equals(c.getName()); } } diff --git a/src/sqlancer/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java similarity index 54% rename from src/sqlancer/schema/AbstractTables.java rename to src/sqlancer/common/schema/AbstractTables.java index 57dfcc903..ff20fde20 100644 --- a/src/sqlancer/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -1,11 +1,11 @@ -package sqlancer.schema; +package sqlancer.common.schema; import java.util.ArrayList; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; -public class AbstractTables, C extends AbstractTableColumn> { +public class AbstractTables, C extends AbstractTableColumn> { private final List tables; private final List columns; @@ -30,13 +30,31 @@ public List getColumns() { return columns; } - public String columnNamesAsString() { - return getColumns().stream().map(t -> t.getTable().getName() + "." + t.getName()) - .collect(Collectors.joining(", ")); - } - public String columnNamesAsString(Function function) { return getColumns().stream().map(function).collect(Collectors.joining(", ")); } + public void addTable(T table) { + if (!this.tables.contains(table)) { + this.tables.add(table); + columns.addAll(table.getColumns()); + } + } + + public void removeTable(T table) { + if (this.tables.contains(table)) { + this.tables.remove(table); + for (C c : table.getColumns()) { + columns.remove(c); + } + } + } + + public boolean isContained(T table) { + return this.tables.contains(table); + } + + public int getSize() { + return this.tables.size(); + } } diff --git a/src/sqlancer/schema/TableIndex.java b/src/sqlancer/common/schema/TableIndex.java similarity index 92% rename from src/sqlancer/schema/TableIndex.java rename to src/sqlancer/common/schema/TableIndex.java index 7d21fbf40..325fe1144 100644 --- a/src/sqlancer/schema/TableIndex.java +++ b/src/sqlancer/common/schema/TableIndex.java @@ -1,4 +1,4 @@ -package sqlancer.schema; +package sqlancer.common.schema; public class TableIndex { diff --git a/src/sqlancer/visitor/BinaryOperation.java b/src/sqlancer/common/visitor/BinaryOperation.java similarity index 78% rename from src/sqlancer/visitor/BinaryOperation.java rename to src/sqlancer/common/visitor/BinaryOperation.java index 26fb81c23..3e9ed9cbf 100644 --- a/src/sqlancer/visitor/BinaryOperation.java +++ b/src/sqlancer/common/visitor/BinaryOperation.java @@ -1,4 +1,4 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; public interface BinaryOperation { diff --git a/src/sqlancer/visitor/NodeVisitor.java b/src/sqlancer/common/visitor/NodeVisitor.java similarity index 50% rename from src/sqlancer/visitor/NodeVisitor.java rename to src/sqlancer/common/visitor/NodeVisitor.java index 393b012dd..68fd73c2a 100644 --- a/src/sqlancer/visitor/NodeVisitor.java +++ b/src/sqlancer/common/visitor/NodeVisitor.java @@ -1,4 +1,4 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; public class NodeVisitor { diff --git a/src/sqlancer/visitor/TernaryOperation.java b/src/sqlancer/common/visitor/TernaryOperation.java similarity index 75% rename from src/sqlancer/visitor/TernaryOperation.java rename to src/sqlancer/common/visitor/TernaryOperation.java index 9ac64e279..fbab64e95 100644 --- a/src/sqlancer/visitor/TernaryOperation.java +++ b/src/sqlancer/common/visitor/TernaryOperation.java @@ -1,4 +1,4 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; public interface TernaryOperation { diff --git a/src/sqlancer/visitor/ToStringVisitor.java b/src/sqlancer/common/visitor/ToStringVisitor.java similarity index 95% rename from src/sqlancer/visitor/ToStringVisitor.java rename to src/sqlancer/common/visitor/ToStringVisitor.java index 50168e4c7..c2385b94d 100644 --- a/src/sqlancer/visitor/ToStringVisitor.java +++ b/src/sqlancer/common/visitor/ToStringVisitor.java @@ -1,8 +1,8 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; import java.util.List; -import sqlancer.visitor.UnaryOperation.OperatorKind; +import sqlancer.common.visitor.UnaryOperation.OperatorKind; public abstract class ToStringVisitor extends NodeVisitor { diff --git a/src/sqlancer/visitor/UnaryOperation.java b/src/sqlancer/common/visitor/UnaryOperation.java similarity index 89% rename from src/sqlancer/visitor/UnaryOperation.java rename to src/sqlancer/common/visitor/UnaryOperation.java index c0b5fe9e4..9d937085e 100644 --- a/src/sqlancer/visitor/UnaryOperation.java +++ b/src/sqlancer/common/visitor/UnaryOperation.java @@ -1,4 +1,4 @@ -package sqlancer.visitor; +package sqlancer.common.visitor; public interface UnaryOperation { diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java new file mode 100644 index 000000000..ae2f74a33 --- /dev/null +++ b/src/sqlancer/databend/DatabendBugs.java @@ -0,0 +1,26 @@ +package sqlancer.databend; + +public final class DatabendBugs { + + public static boolean bug9018; // https://github.com/datafuselabs/databend/issues/9018 + public static boolean bug9162; // https://github.com/datafuselabs/databend/issues/9162 + public static boolean bug9163; // https://github.com/datafuselabs/databend/issues/9163 + public static boolean bug9164 = true; // https://github.com/datafuselabs/databend/issues/9164 + public static boolean bug9196 = true; // https://github.com/datafuselabs/databend/issues/9196 + public static boolean bug9232 = true; // https://github.com/datafuselabs/databend/issues/9232 + public static boolean bug9224 = true; // https://github.com/datafuselabs/databend/issues/9224 + public static boolean bug9226 = true; // https://github.com/datafuselabs/databend/issues/9226 + public static boolean bug9234 = true; // https://github.com/datafuselabs/databend/issues/9234 + public static boolean bug9235 = true; // https://github.com/datafuselabs/databend/issues/9235 + public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 + public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 + public static boolean bug9806 = true; // https://github.com/datafuselabs/databend/issues/9806 + public static boolean bug15568 = true; // https://github.com/datafuselabs/databend/issues/15568 + public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 + public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 + public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 + public static boolean bug19773 = true; // https://github.com/databendlabs/databend/issues/19773 + + private DatabendBugs() { + } +} diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java new file mode 100644 index 000000000..3e056d003 --- /dev/null +++ b/src/sqlancer/databend/DatabendErrors.java @@ -0,0 +1,102 @@ +package sqlancer.databend; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class DatabendErrors { + + private DatabendErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Division by zero"); + errors.add("divided by zero"); + errors.add("/ by zero"); + errors.add("ORDER BY position"); + errors.add("GROUP BY position"); + errors.add("no overload satisfies `not(Float64 NULL)`"); // TODO databend不允许出现not(float)，而a/b为float + errors.add("no overload satisfies `not(Float64)`"); + errors.add("number overflowed while evaluating function"); // 表达式数值溢出 + errors.add("Unable to get field named"); + errors.add("no overload satisfies `and_filters"); + if (DatabendBugs.bug9162) { + errors.add("downcast column error"); + } + if (DatabendBugs.bug9018) { + errors.add("index out of bounds"); + } + if (DatabendBugs.bug9163) { + errors.add("validity must be equal to the array's length"); + } + if (DatabendBugs.bug9224) { + errors.add("Can't cast column from nullable data into non-nullable type"); + } + if (DatabendBugs.bug9234) { + errors.add("called `Option::unwrap()` on a `None` value"); + } + if (DatabendBugs.bug9264) { + errors.add("assertion failed: offset + length <= self.length"); + } + if (DatabendBugs.bug9806) { + errors.add("segment pruning failure"); + } + if (DatabendBugs.bug15568) { + errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); + } + if (DatabendBugs.bug19773) { + errors.add("failed to downcast column Decimal128"); + errors.add("Decimal(DecimalSize { precision: 38"); + errors.add("_eager_final_count"); + } + + /* + * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE + * databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL + * DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES + * ('067596','19'), ('', '87'); + */ + errors.add("Can't cast column from null into non-nullable type"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Division by zero"); + errors.add("/ by zero"); + errors.add("violates not-null constraint"); + errors.add("number overflowed while evaluating function `"); // 不能在int16类型column上插入int64的数据 + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Division by zero"); + errors.add("/ by zero"); + errors.add("Can't cast column from null into non-nullable type"); + errors.add("GROUP BY position"); + errors.add("GROUP BY items can't contain aggregate functions or window functions"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); + } + +} diff --git a/src/sqlancer/databend/DatabendExpectedValueVisitor.java b/src/sqlancer/databend/DatabendExpectedValueVisitor.java new file mode 100644 index 000000000..bfa6208a3 --- /dev/null +++ b/src/sqlancer/databend/DatabendExpectedValueVisitor.java @@ -0,0 +1,152 @@ +package sqlancer.databend; + +import java.util.List; + +import sqlancer.databend.ast.DatabendAlias; +import sqlancer.databend.ast.DatabendBetweenOperation; +import sqlancer.databend.ast.DatabendBinaryOperation; +import sqlancer.databend.ast.DatabendColumnReference; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendFunctionOperation; +import sqlancer.databend.ast.DatabendInOperation; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendOrderByTerm; +import sqlancer.databend.ast.DatabendPostFixText; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; + +public class DatabendExpectedValueVisitor { + + protected final StringBuilder sb = new StringBuilder(); + + private void print(DatabendExpression expr) { + sb.append(DatabendToStringVisitor.asString(expr)); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + public void visit(DatabendExpression expr) { + assert expr != null; + if (expr instanceof DatabendColumnReference) { + visit((DatabendColumnReference) expr); + } else if (expr instanceof DatabendUnaryPostfixOperation) { + visit((DatabendUnaryPostfixOperation) expr); + } else if (expr instanceof DatabendUnaryPrefixOperation) { + visit((DatabendUnaryPrefixOperation) expr); + } else if (expr instanceof DatabendBinaryOperation) { + visit((DatabendBinaryOperation) expr); + } else if (expr instanceof DatabendTableReference) { + visit((DatabendTableReference) expr); + } else if (expr instanceof DatabendFunctionOperation) { + visit((DatabendFunctionOperation) expr); + } else if (expr instanceof DatabendBetweenOperation) { + visit((DatabendBetweenOperation) expr); + } else if (expr instanceof DatabendInOperation) { + visit((DatabendInOperation) expr); + } else if (expr instanceof DatabendOrderByTerm) { + visit((DatabendOrderByTerm) expr); + } else if (expr instanceof DatabendAlias) { + visit((DatabendAlias) expr); + } else if (expr instanceof DatabendPostFixText) { + visit((DatabendPostFixText) expr); + } else if (expr instanceof DatabendConstant) { + visit((DatabendConstant) expr); + } else if (expr instanceof DatabendSelect) { + visit((DatabendSelect) expr); + } else if (expr instanceof DatabendJoin) { + visit((DatabendJoin) expr); + } else { + throw new AssertionError(expr); + } + } + + public void visit(DatabendColumnReference c) { + print(c); + } + + public void visit(DatabendUnaryPostfixOperation op) { + print(op); + visit(op.getExpr()); + } + + public void visit(DatabendUnaryPrefixOperation op) { + print(op); + visit(op.getExpr()); + } + + public void visit(DatabendBinaryOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(DatabendTableReference t) { + print(t); + } + + public void visit(DatabendFunctionOperation fun) { + print(fun); + visit(fun.getArgs()); + } + + public void visit(List expressions) { + for (DatabendExpression expression : expressions) { + visit(expression); + } + } + + public void visit(DatabendBetweenOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getMiddle()); + visit(op.getRight()); + } + + public void visit(DatabendInOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(DatabendOrderByTerm op) { + print(op); + visit(op.getExpr()); + } + + public void visit(DatabendAlias op) { + print(op); + visit(op.getExpr()); + } + + public void visit(DatabendPostFixText postFixText) { + print(postFixText); + visit(postFixText.getExpr()); + } + + public void visit(DatabendConstant constant) { + print(constant); + } + + public void visit(DatabendSelect select) { + print(select.getWhereClause()); + } + + public void visit(DatabendJoin join) { + print(join.getOnCondition()); + } + + public String get() { + return sb.toString(); + } + + public static String asExpectedValues(DatabendExpression expr) { + DatabendExpectedValueVisitor v = new DatabendExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + +} diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java new file mode 100644 index 000000000..d38ee0d59 --- /dev/null +++ b/src/sqlancer/databend/DatabendOptions.java @@ -0,0 +1,90 @@ +package sqlancer.databend; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(commandDescription = "Databend") +public class DatabendOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3307; + + @Parameter(names = "--test-collate", arity = 1) + public boolean testCollate = true; + + @Parameter(names = "--test-check", description = "Allow generating CHECK constraints in tables", arity = 1) + public boolean testCheckConstraints = true; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions = true; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions", arity = 1) + public boolean testBetween = true; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-timestamp-constants", description = "Allow generating TIMESTAMP constants", arity = 1) + public boolean testTimestampConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--test-indexes", description = "Allow explicit (i.e. CREATE INDEX) and implicit (i.e., UNIQUE and PRIMARY KEY) indexes", arity = 1) + public boolean testIndexes = true; + + @Parameter(names = "--test-rowid", description = "Test tables' rowid columns", arity = 1) + public boolean testRowid = true; + + @Parameter(names = "--max-num-views", description = "The maximum number of views that can be generated for a database", arity = 1) + public int maxNumViews = 1; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(DatabendOracleFactory.QUERY_PARTITIONING); + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/databend/DatabendOracleFactory.java b/src/sqlancer/databend/DatabendOracleFactory.java new file mode 100644 index 000000000..ea66bc353 --- /dev/null +++ b/src/sqlancer/databend/DatabendOracleFactory.java @@ -0,0 +1,93 @@ +package sqlancer.databend; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; +import sqlancer.databend.test.DatabendPivotedQuerySynthesisOracle; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningDistinctTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningGroupByTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningHavingTester; + +public enum DatabendOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with(DatabendErrors.getGroupByErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + PQS { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws Exception { + return new DatabendPivotedQuerySynthesisOracle(globalState); + } + } + +} diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java new file mode 100644 index 000000000..df7802027 --- /dev/null +++ b/src/sqlancer/databend/DatabendProvider.java @@ -0,0 +1,152 @@ +package sqlancer.databend; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.gen.DatabendDeleteGenerator; +import sqlancer.databend.gen.DatabendInsertGenerator; +import sqlancer.databend.gen.DatabendRandomQuerySynthesizer; +import sqlancer.databend.gen.DatabendTableGenerator; +import sqlancer.databend.gen.DatabendViewGenerator; + +@AutoService(DatabaseProvider.class) +public class DatabendProvider extends SQLProviderAdapter { + + public DatabendProvider() { + super(DatabendGlobalState.class, DatabendOptions.class); + } + + public enum Action implements AbstractAction { + + INSERT(DatabendInsertGenerator::getQuery), DELETE(DatabendDeleteGenerator::generate), + // TODO 等待databend实现update + // UPDATE(DatabendUpdateGenerator::getQuery), // + CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> { + ExpectedErrors errors = new ExpectedErrors(); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addGroupByErrors(errors); + return new SQLQueryAdapter( + "EXPLAIN " + DatabendToStringVisitor + .asString(DatabendRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + errors); + }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(DatabendGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(DatabendGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case EXPLAIN: + return r.getInteger(0, 2); + // TODO 等待databend实现update && delete + // case UPDATE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + case CREATE_VIEW: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + default: + throw new AssertionError(a); + } + } + + public static class DatabendGlobalState extends SQLGlobalState { + + @Override + protected DatabendSchema readSchema() throws SQLException { + return DatabendSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(DatabendGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(3, 4); i++) { + boolean success; + do { + SQLQueryAdapter qt = new DatabendTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + DatabendProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); // 增删改一些数据（按权重随机选取算法） + } + + @Override + public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = DatabendOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = DatabendOptions.DEFAULT_PORT; + } + String databaseName = globalState.getDatabaseName(); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + s.execute("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + s.execute("USE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + } + if (DatabendBugs.bug15569) { + con.close(); + String urlWithRetry = String.format( + "jdbc:mysql://%s:%d/%s?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true&autoReconnect=true", + host, port, databaseName); + con = DriverManager.getConnection(urlWithRetry, username, password); + } + + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "databend"; // 用于DatabendOptions + } + +} diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java new file mode 100644 index 000000000..89738a1f3 --- /dev/null +++ b/src/sqlancer/databend/DatabendSchema.java @@ -0,0 +1,383 @@ +package sqlancer.databend; + +import static sqlancer.databend.DatabendSchema.DatabendDataType.INT; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.ast.DatabendConstant; + +public class DatabendSchema extends AbstractSchema { + + public enum DatabendDataType { + + INT, VARCHAR, BOOLEAN, FLOAT, NULL, DATE, TIMESTAMP; + + public static DatabendDataType getRandomWithoutNull() { + DatabendDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DatabendDataType.NULL); + return dt; + } + + public static DatabendDataType getRandomWithoutNullAndVarchar() { + DatabendDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DatabendDataType.NULL || dt == DatabendDataType.VARCHAR); + return dt; + } + + } + + public static class DatabendCompositeDataType { + + private final DatabendDataType dataType; + + private final int size; + + public DatabendCompositeDataType(DatabendDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public DatabendDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static DatabendCompositeDataType getRandomWithoutNull() { + DatabendDataType type = DatabendDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case BOOLEAN: + case VARCHAR: + case DATE: + case TIMESTAMP: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new DatabendCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return Randomly.fromOptions("BIGINT", "INT64"); + case 4: + return Randomly.fromOptions("INT", "INT32"); + case 2: + return Randomly.fromOptions("SMALLINT", "INT16"); + case 1: + return Randomly.fromOptions("TINYINT", "INT8"); + default: + throw new AssertionError(size); + } + case VARCHAR: + return Randomly.fromOptions("VARCHAR"); + case FLOAT: + switch (size) { + case 8: + return Randomly.fromOptions("DOUBLE"); + case 4: + return Randomly.fromOptions("FLOAT"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN", "BOOL"); + case DATE: + return Randomly.fromOptions("DATE"); + case TIMESTAMP: + return Randomly.fromOptions("TIMESTAMP", "DATETIME"); + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class DatabendColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public DatabendColumn(String name, DatabendCompositeDataType columnType, boolean isPrimaryKey, + boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + @Override + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class DatabendTables extends AbstractTables { + + public DatabendTables(List tables) { + super(tables); + } + + public DatabendRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String rowValueQuery = String.format("SELECT %s FROM %s ORDER BY 1 LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery(rowValueQuery); + if (!rs.next()) { + throw new IgnoreMeException(); + // throw new AssertionError("could not find random row " + rowValueQuery + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + DatabendColumn column = getColumns().get(i); + int columnIndex = rs.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + DatabendConstant constant; + if (rs.getString(columnIndex) == null) { + constant = DatabendConstant.createNullConstant(); + } else { + switch (column.getType().getPrimitiveDataType()) { + case INT: + constant = DatabendConstant.createIntConstant(rs.getLong(columnIndex)); + break; + case BOOLEAN: + constant = DatabendConstant.createBooleanConstant(rs.getBoolean(columnIndex)); + break; + case VARCHAR: + constant = DatabendConstant.createStringConstant(rs.getString(columnIndex)); + break; + case DATE: + constant = DatabendConstant.createDateConstant(rs.getLong(columnIndex)); + break; + case TIMESTAMP: + constant = DatabendConstant.createTimestampConstant(rs.getLong(columnIndex)); + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !rs.next(); + return new DatabendRowValue(this, values); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + + } + + public static class DatabendRowValue extends AbstractRowValue { + + DatabendRowValue(DatabendTables tables, Map values) { + super(tables, values); + } + + } + + public DatabendSchema(List databaseTables) { + super(databaseTables); + } + + public DatabendTables getRandomTableNonEmptyTables() { + return new DatabendTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public DatabendTables getRandomTableNonEmptyAndViewTables() { + List tables = getDatabaseTables().stream().filter(t -> !t.isView()).collect(Collectors.toList()); + tables = Randomly.nonEmptySubset(tables); + return new DatabendTables(tables); + } + + private static DatabendCompositeDataType getColumnType(String typeString) { + if (typeString.startsWith("DECIMAL")) { // Ugly hack + return new DatabendCompositeDataType(DatabendDataType.FLOAT, 8); + } + if (typeString.startsWith("Nullable")) { // Ugly hack + String substring = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + return getColumnTypeNormalCases(substring); + } + return getColumnTypeNormalCases(typeString); + } + + private static DatabendCompositeDataType getColumnTypeNormalCases(String typeString) { + DatabendDataType primitiveType; + int size = -1; + switch (typeString.toUpperCase()) { + case "BOOLEAN": + case "BOOL": + primitiveType = DatabendDataType.BOOLEAN; + size = 1; + break; + case "TINYINT": + case "INT8": + primitiveType = INT; + size = 1; + break; + case "SMALLINT": + case "INT16": + primitiveType = INT; + size = 2; + break; + case "INT": + case "INT32": + primitiveType = INT; + size = 4; + break; + case "BIGINT": + case "INT64": + primitiveType = INT; + size = 8; + break; + case "FLOAT": + case "FLOAT32": + primitiveType = DatabendDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + case "FLOAT64": + primitiveType = DatabendDataType.FLOAT; + size = 8; + break; + case "DATE": + primitiveType = DatabendDataType.DATE; + break; + case "TIMESTAMP": + primitiveType = DatabendDataType.TIMESTAMP; + break; + case "VARCHAR": + case "STRING": + primitiveType = DatabendDataType.VARCHAR; + break; + case "NULL": + primitiveType = DatabendDataType.NULL; + break; + case "INTERVAL": + throw new IgnoreMeException(); + // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP + // '1969-12-29 06:50:27')) + default: + throw new AssertionError(typeString); + } + return new DatabendCompositeDataType(primitiveType, size); + } + + public static class DatabendTable extends AbstractRelationalTable { + + public DatabendTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static DatabendSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con, databaseName); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName, databaseName); + boolean isView = matchesViewName(tableName); + DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); + for (DatabendColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new DatabendSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { + List tableNames = null; + tableNames = new ArrayList<>(); + + final String sqlStatement = String.format( + "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ", + databaseName); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(sqlStatement)) { + while (rs.next()) { + tableNames.add(rs.getString("table_name")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = '%s' and table_name ='%s'", + databaseName, tableName))) { + try { // 没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + if (dataType.contains("NULL")) { + dataType = dataType.substring(0, dataType.indexOf(' ')); + } + boolean isNullable = rs.getBoolean("is_nullable"); + // boolean isPrimaryKey = rs.getString("pk").contains("true"); + boolean isPrimaryKey = false; // 没找到主键元数据 + DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, + isNullable); + columns.add(c); + } + } catch (Exception e) { + System.out.println("TableColumns->SQLException：Not a navigable ResultSet"); + } + } + } + + return columns; + } + +} diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java new file mode 100644 index 000000000..05d09d7f2 --- /dev/null +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -0,0 +1,99 @@ +package sqlancer.databend; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; + +public class DatabendToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(DatabendExpression expr) { + if (expr instanceof DatabendConstant) { + visit((DatabendConstant) expr); + } else if (expr instanceof DatabendSelect) { + visit((DatabendSelect) expr); + } else if (expr instanceof DatabendJoin) { + visit((DatabendJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DatabendJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DatabendConstant constant) { + sb.append(constant.toString()); + } + + // private void visitFromList(List> fromList) { + // for (int i = 0; i < fromList.size(); i++) { + // if (i != 0) { + // sb.append(" INNER JOIN "); + // } + // visit(fromList.get(i)); + // } + // } + + private void visit(DatabendSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(DatabendExpression expr) { + DatabendToStringVisitor visitor = new DatabendToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendAggregateOperation.java b/src/sqlancer/databend/ast/DatabendAggregateOperation.java new file mode 100644 index 000000000..5070991d8 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendAggregateOperation.java @@ -0,0 +1,46 @@ +package sqlancer.databend.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.databend.DatabendSchema; + +public class DatabendAggregateOperation extends + DatabendFunctionOperation implements DatabendExpression { + public DatabendAggregateOperation(List args, DatabendAggregateFunction func) { + super(args, func); + } + + public enum DatabendAggregateFunction { + MAX(1), MIN(1), AVG(1, DatabendSchema.DatabendDataType.INT, DatabendSchema.DatabendDataType.FLOAT), COUNT(1), + SUM(1, DatabendSchema.DatabendDataType.INT, DatabendSchema.DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), + COVAR_SAMP(2); + // , *_IF, *_DISTINCT + + private int nrArgs; + private DatabendSchema.DatabendDataType[] dataTypes; + + DatabendAggregateFunction(int nrArgs, DatabendSchema.DatabendDataType... dataTypes) { + this.nrArgs = nrArgs; + this.dataTypes = dataTypes.clone(); + } + + public static DatabendAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public DatabendSchema.DatabendDataType getRandomType() { + if (dataTypes.length == 0) { + return Randomly.fromOptions(DatabendSchema.DatabendDataType.values()); + } else { + return Randomly.fromOptions(dataTypes); + } + } + + public int getNrArgs() { + return nrArgs; + } + + } + +} diff --git a/src/sqlancer/databend/ast/DatabendAlias.java b/src/sqlancer/databend/ast/DatabendAlias.java new file mode 100644 index 000000000..5f6827409 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendAlias.java @@ -0,0 +1,9 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DatabendAlias extends NewAliasNode implements DatabendExpression { + public DatabendAlias(DatabendExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/databend/ast/DatabendBetweenOperation.java b/src/sqlancer/databend/ast/DatabendBetweenOperation.java new file mode 100644 index 000000000..a6a0134bf --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBetweenOperation.java @@ -0,0 +1,39 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendBetweenOperation extends NewBetweenOperatorNode implements DatabendExpression { + public DatabendBetweenOperation(DatabendExpression left, DatabendExpression middle, DatabendExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } + + public DatabendExpression getLeftExpr() { + return left; + } + + public DatabendExpression getMiddleExpr() { + return middle; + } + + public DatabendExpression getRightExpr() { + return right; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendBinaryComparisonOperation leftComparison = new DatabendBinaryComparisonOperation(getMiddleExpr(), + getLeftExpr(), DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator.LESS_EQUALS); + DatabendBinaryComparisonOperation rightComparison = new DatabendBinaryComparisonOperation(getLeftExpr(), + getRightExpr(), DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator.LESS_EQUALS); + return new DatabendBinaryLogicalOperation(leftComparison, rightComparison, + DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator.AND).getExpectedValue(); + } + + @Override + public DatabendSchema.DatabendDataType getExpectedType() { + return DatabendSchema.DatabendDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java new file mode 100644 index 000000000..5554cc474 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -0,0 +1,100 @@ +package sqlancer.databend.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode + implements DatabendExpression { + + public DatabendBinaryArithmeticOperation(DatabendExpression left, DatabendExpression right, + BinaryOperatorNode.Operator op) { + super(left, right, op); + } + + public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADDITION("+") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l + r); + } + }, + SUBTRACTION("-") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { // TODO databend不允许出现not(float)，而a/b为float + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + } + }, + MODULO("%") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + } + }; + + private final String textRepresentation; + + DatabendBinaryArithmeticOperator(String text) { + textRepresentation = text; + } + + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + + public DatabendConstant applyOperation(DatabendConstant left, DatabendConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + long leftVal = left.cast(DatabendDataType.INT).asInt(); + long rightVal = right.cast(DatabendDataType.INT).asInt(); + return DatabendConstant.createIntConstant(op.apply(leftVal, rightVal)); + } + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + public DatabendExpression getLeftExpr() { + return super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return super.getRight(); + } + + public DatabendBinaryArithmeticOperator getOp() { + return (DatabendBinaryArithmeticOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = getLeftExpr().getExpectedValue(); + DatabendConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.INT; + } + +} diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java new file mode 100644 index 000000000..12fc8c376 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -0,0 +1,139 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode + implements DatabendExpression { + + public DatabendBinaryComparisonOperation(DatabendExpression left, DatabendExpression right, + DatabendBinaryComparisonOperator op) { + super(left, right, op); + } + + public DatabendExpression getLeftExpression() { + return super.getLeft(); + } + + public DatabendExpression getRightExpression() { + return super.getRight(); + } + + public DatabendBinaryComparisonOperator getOp() { + return (DatabendBinaryComparisonOperator) op; + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftExpectedValue = getLeftExpression().getExpectedValue(); + DatabendConstant rightExpectedValue = getRightExpression().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + + public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return left.isEquals(right); + } + }, + NOT_EQUALS("!=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean()) { + return DatabendConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + IS_DISTINCT("IS DISTINCT FROM") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return DatabendConstant.createBooleanConstant(!IS_NOT_DISTINCT.apply(left, right).asBoolean()); + } + }, + IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + if (left.isNull()) { + return DatabendConstant.createBooleanConstant(right.isNull()); + } else if (right.isNull()) { + return DatabendConstant.createBooleanConstant(false); + } else { + return left.isEquals(right); + } + } + }, + LESS("<") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return left.isLessThan(right); + } + }, + LESS_EQUALS("<=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isLessThan = left.isLessThan(right); + if (isLessThan.isBoolean() && !isLessThan.asBoolean()) { + return left.isEquals(right); + } else { + return isLessThan; + } + } + }, + GREATER(">") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean() && isEquals.asBoolean()) { + return DatabendConstant.createBooleanConstant(false); + } else { + DatabendConstant less = left.isLessThan(right); + if (less.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(!less.asBoolean()); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean() && isEquals.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } else { + DatabendConstant less = left.isLessThan(right); + if (less.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(!less.asBoolean()); + } + } + }; + + private final String textRepresentation; + + DatabendBinaryComparisonOperator(String text) { + textRepresentation = text; + } + + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + +} diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java new file mode 100644 index 000000000..fd51e1eca --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -0,0 +1,115 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode + implements DatabendExpression { + + public DatabendBinaryLogicalOperation(DatabendExpression left, DatabendExpression right, + DatabendBinaryLogicalOperator op) { + super(left, right, op); + } + + public DatabendExpression getLeftExpr() { + return super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return super.getRight(); + } + + public DatabendBinaryLogicalOperator getOp() { + return (DatabendBinaryLogicalOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = getLeftExpr().getExpectedValue(); + DatabendConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; + } + + public enum DatabendBinaryLogicalOperator implements BinaryOperatorNode.Operator { + AND("AND", "and") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant leftVal = left.cast(DatabendDataType.BOOLEAN); + DatabendConstant rightVal = right.cast(DatabendDataType.BOOLEAN); + assert leftVal.isNull() || leftVal.isBoolean() : leftVal + "不是NULL也不是Boolean类型"; + assert rightVal.isNull() || rightVal.isBoolean() : rightVal + "不是NULL也不是Boolean类型"; + if (leftVal.isNull()) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + if (rightVal.asBoolean()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(false); + } + } + } else if (!leftVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(false); + } + assert leftVal.asBoolean(); + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(rightVal.asBoolean()); + } + } + }, + OR("OR", "or") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant leftVal = left.cast(DatabendDataType.BOOLEAN); + DatabendConstant rightVal = right.cast(DatabendDataType.BOOLEAN); + assert leftVal.isNull() || leftVal.isBoolean() : leftVal + "不是NULL也不是Boolean类型"; + assert rightVal.isNull() || rightVal.isBoolean() : rightVal + "不是NULL也不是Boolean类型"; + if (leftVal.isBoolean() && leftVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } + if (rightVal.isBoolean() && rightVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } + if (leftVal.isNull() || rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(false); + } + }; + + private final String[] textRepresentations; + + DatabendBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public DatabendBinaryLogicalOperator getRandomOp() { + return Randomly.fromOptions(values()); + } + + public static DatabendBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + + } + +} diff --git a/src/sqlancer/databend/ast/DatabendBinaryOperation.java b/src/sqlancer/databend/ast/DatabendBinaryOperation.java new file mode 100644 index 000000000..c9fab806e --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryOperation.java @@ -0,0 +1,12 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DatabendBinaryOperation extends NewBinaryOperatorNode implements DatabendExpression { + public DatabendBinaryOperation(DatabendExpression left, DatabendExpression right, + BinaryOperatorNode.Operator operator) { + super(left, right, operator); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendCastOperation.java b/src/sqlancer/databend/ast/DatabendCastOperation.java new file mode 100644 index 000000000..07e7d7f9c --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendCastOperation.java @@ -0,0 +1,40 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendCastOperation extends NewUnaryPostfixOperatorNode + implements DatabendExpression { + + DatabendDataType type; + + public DatabendCastOperation(DatabendExpression expr, DatabendCompositeDataType type) { + super(expr, new BinaryOperatorNode.Operator() { + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + this.type = type.getPrimitiveDataType(); + } + + DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type); + } + + @Override + public DatabendDataType getExpectedType() { + return type; + } +} diff --git a/src/sqlancer/databend/ast/DatabendColumnReference.java b/src/sqlancer/databend/ast/DatabendColumnReference.java new file mode 100644 index 000000000..4b53ef4df --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendColumnReference extends ColumnReferenceNode + implements DatabendExpression { + public DatabendColumnReference(DatabendSchema.DatabendColumn column) { + super(column); + } +} diff --git a/src/sqlancer/databend/ast/DatabendColumnValue.java b/src/sqlancer/databend/ast/DatabendColumnValue.java new file mode 100644 index 000000000..8b1992880 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendColumnValue.java @@ -0,0 +1,31 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendColumnValue extends ColumnReferenceNode + implements DatabendExpression { + + private final DatabendConstant expectedValue; + + public DatabendColumnValue(DatabendColumn column, DatabendConstant value) { + super(column); + this.expectedValue = value; + } + + @Override + public DatabendConstant getExpectedValue() { + return expectedValue; + } + + @Override + public DatabendDataType getExpectedType() { + return getColumn().getType().getPrimitiveDataType(); + } + + public static DatabendColumnValue create(DatabendColumn column, DatabendConstant value) { + return new DatabendColumnValue(column, value); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java new file mode 100644 index 000000000..919942a8a --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -0,0 +1,493 @@ +package sqlancer.databend.ast; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public abstract class DatabendConstant implements DatabendExpression { + + private DatabendConstant() { + } + + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public abstract DatabendConstant cast(DatabendDataType dataType); + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + protected Timestamp truncateTimestamp(long val) { + // Databend supports `date` and `timestamp` type where the year cannot exceed `9999`, + // the value is truncated to ensure generate legitimate `date` and `timestamp` value. + long t = val % 253380000000000L; + return new Timestamp(t); + } + + public abstract DatabendConstant isEquals(DatabendConstant rightVal); + + public abstract DatabendConstant isLessThan(DatabendConstant rightVal); + + // public abstract String getTextRepresentation(); + + public static class DatabendNullConstant extends DatabendConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.NULL; + } + + // @Override + // public DatabendConstant getExpectedValue() { + // return super.getExpectedValue(); + // } + } + + public static class DatabendIntConstant extends DatabendConstant { + + private final long value; + + public DatabendIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case BOOLEAN: + return new DatabendBooleanConstant(value != 0); + case INT: + return this; + case VARCHAR: + return new DatabendStringConstant(String.valueOf(value)); + case DATE: + return new DatabendDateConstant(value); + case TIMESTAMP: + return new DatabendTimestampConstant(value); + default: + return null; + } + } + + @Override + public long asInt() { + return value; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value == rightVal.asInt()); + } else { + throw new AssertionError(rightVal); + } + + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.INT; + } + } + + public static class DatabendFloatConstant extends DatabendConstant { + + private final double value; + + public DatabendFloatConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public boolean isFloat() { + return true; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "3.40282347e+38"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-3.40282347e+38"; + } + + return String.valueOf(value); + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case FLOAT: + return this; + case INT: + return DatabendConstant.createIntConstant((long) value); + case BOOLEAN: + return DatabendConstant.createBooleanConstant(value != 0); + case VARCHAR: + return DatabendConstant.createStringConstant(String.valueOf(value)); + default: + return null; + } + } + + @Override + public double asFloat() { + return value; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + } + + public static class DatabendStringConstant extends DatabendConstant { + + private final String value; + + public DatabendStringConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + @Override + public String asString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case VARCHAR: + return this; + case INT: + try { + return new DatabendIntConstant(Long.parseLong(value)); + } catch (NumberFormatException e) { + return new DatabendIntConstant(-1); + } + case BOOLEAN: + if ("false".contentEquals(value.toLowerCase())) { + return new DatabendBooleanConstant(false); + } else if ("true".contentEquals(value.toLowerCase())) { + return new DatabendBooleanConstant(true); + } else { + throw new AssertionError(String.format("string: %s, cannot be forced to boolean", value)); + } + case FLOAT: + try { + return new DatabendFloatConstant(Double.parseDouble(value)); + } catch (NumberFormatException e) { + return new DatabendFloatConstant(-1); + } + default: + return null; + } + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isString()) { + return DatabendConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + // TODO 可以比较 date和timestamp类型，待添加 + throw new AssertionError(rightVal); + } + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isString()) { + return DatabendConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + } + + public static class DatabendDateConstant extends DatabendConstant { + + public String textRepr; + + public DatabendDateConstant(long val) { + Timestamp timestamp = truncateTimestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return null; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return null; + } + } + + public static class DatabendTimestampConstant extends DatabendConstant { + + public String textRepr; + + public DatabendTimestampConstant(long val) { + Timestamp timestamp = truncateTimestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return null; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return null; + } + } + + public static class DatabendBooleanConstant extends DatabendConstant { + + private final boolean value; + + public DatabendBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case BOOLEAN: + return this; + case INT: + return new DatabendIntConstant(value ? 1 : 0); + case FLOAT: + return new DatabendFloatConstant(value ? 1 : 0); + case VARCHAR: + return new DatabendStringConstant(value ? "1" : "0"); + default: + return null; + } + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return DatabendConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else { + throw new AssertionError(rightVal); + } + + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return DatabendConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } else { + throw new AssertionError(rightVal); + } + } + + } + + public static DatabendConstant createStringConstant(String text) { + return new DatabendStringConstant(text); + } + + public static DatabendConstant createFloatConstant(double val) { + return new DatabendFloatConstant(val); + } + + public static DatabendConstant createIntConstant(long val) { + return new DatabendIntConstant(val); + } + + public static DatabendConstant createNullConstant() { + return new DatabendNullConstant(); + } + + public static DatabendConstant createBooleanConstant(boolean val) { + return new DatabendBooleanConstant(val); + } + + public static DatabendConstant createDateConstant(long integer) { + return new DatabendDateConstant(integer); + } + + public static DatabendConstant createTimestampConstant(long integer) { + return new DatabendTimestampConstant(integer); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendExpression.java b/src/sqlancer/databend/ast/DatabendExpression.java new file mode 100644 index 000000000..d71e64ed7 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendExpression.java @@ -0,0 +1,16 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public interface DatabendExpression extends Expression { + + default DatabendDataType getExpectedType() { + return null; + } + + default DatabendConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/databend/ast/DatabendFunctionOperation.java b/src/sqlancer/databend/ast/DatabendFunctionOperation.java new file mode 100644 index 000000000..69cb16175 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendFunctionOperation.java @@ -0,0 +1,11 @@ +package sqlancer.databend.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DatabendFunctionOperation extends NewFunctionNode implements DatabendExpression { + public DatabendFunctionOperation(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/databend/ast/DatabendInOperation.java b/src/sqlancer/databend/ast/DatabendInOperation.java new file mode 100644 index 000000000..f382aae04 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendInOperation.java @@ -0,0 +1,52 @@ +package sqlancer.databend.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendInOperation extends NewInOperatorNode implements DatabendExpression { + + private final DatabendExpression leftExpr; + private final List rightExpr; + + public DatabendInOperation(DatabendExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + this.leftExpr = left; + this.rightExpr = right; + } + + @Override + public DatabendSchema.DatabendDataType getExpectedType() { + return DatabendSchema.DatabendDataType.BOOLEAN; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = leftExpr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return DatabendConstant.createNullConstant(); + } + boolean isNull = false; + for (DatabendExpression expr : rightExpr) { + DatabendConstant rightValue = expr.getExpectedValue(); + if (rightValue == null) { + return null; + } + if (rightValue.isNull()) { + isNull = true; + } else if (rightValue.isEquals(leftValue).isBoolean() && rightValue.isEquals(leftValue).asBoolean()) { + return DatabendConstant.createBooleanConstant(!isNegated()); + } + } + + if (isNull) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(isNegated()); + } + } +} diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java new file mode 100644 index 000000000..9386c1fe1 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -0,0 +1,130 @@ +package sqlancer.databend.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; + +public class DatabendJoin implements DatabendExpression, Join { + + private final DatabendTableReference leftTable; + private final DatabendTableReference rightTable; + private final JoinType joinType; + private DatabendExpression onCondition; + private OuterType outerType; + + public enum JoinType { + INNER, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public DatabendJoin(DatabendTableReference leftTable, DatabendTableReference rightTable, JoinType joinType, + DatabendExpression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public DatabendExpression getOnCondition() { + return onCondition; + } + + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public OuterType getOuterType() { + return outerType; + } + + public static List getJoins(List tableList, DatabendGlobalState globalState) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + DatabendTableReference leftTable = tableList.remove(0); + DatabendTableReference rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState) + .setColumns(columns); + + switch (JoinType.getRandom()) { + case INNER: + joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + break; + case NATURAL: + joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); + break; + case LEFT: + joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + break; + case RIGHT: + joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DatabendJoin createRightOuterJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { + return new DatabendJoin(left, right, JoinType.RIGHT, predicate); + } + + public static DatabendJoin createLeftOuterJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { + return new DatabendJoin(left, right, JoinType.LEFT, predicate); + } + + public static DatabendJoin createInnerJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { + return new DatabendJoin(left, right, JoinType.INNER, predicate); + } + + public static DatabendJoin createNaturalJoin(DatabendTableReference left, DatabendTableReference right, + OuterType naturalJoinType) { + DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); + join.setOuterType(naturalJoinType); + return join; + } + + @Override + public void setOnClause(DatabendExpression onClause) { + onCondition = onClause; + } +} diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java new file mode 100644 index 000000000..f40d804bb --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -0,0 +1,62 @@ +package sqlancer.databend.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendLikeOperation extends NewBinaryOperatorNode implements DatabendExpression { + + public DatabendLikeOperation(DatabendExpression left, DatabendExpression right, DatabendLikeOperator op) { + super(left, right, op); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; + } + + public DatabendExpression getLeftExpr() { + return super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return super.getRight(); + } + + public DatabendLikeOperator getOp() { + return (DatabendLikeOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftVal = getLeftExpr().getExpectedValue(); + DatabendConstant rightVal = getRightExpr().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + if (leftVal.isNull() || rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + boolean result = LikeImplementationHelper.match(leftVal.asString(), rightVal.asString(), 0, 0, true); + return DatabendConstant.createBooleanConstant(result); + } + } + + public enum DatabendLikeOperator implements BinaryOperatorNode.Operator { + LIKE_OPERATOR("LIKE", "like"); + + private final String[] textRepresentations; + + DatabendLikeOperator(String... text) { + textRepresentations = text.clone(); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + } + +} diff --git a/src/sqlancer/databend/ast/DatabendOrderByTerm.java b/src/sqlancer/databend/ast/DatabendOrderByTerm.java new file mode 100644 index 000000000..7492c7c33 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendOrderByTerm.java @@ -0,0 +1,9 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class DatabendOrderByTerm extends NewOrderingTerm implements DatabendExpression { + public DatabendOrderByTerm(DatabendExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/databend/ast/DatabendPostFixText.java b/src/sqlancer/databend/ast/DatabendPostFixText.java new file mode 100644 index 000000000..8f854fbe1 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendPostFixText.java @@ -0,0 +1,9 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DatabendPostFixText extends NewPostfixTextNode implements DatabendExpression { + public DatabendPostFixText(DatabendExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/databend/ast/DatabendSelect.java b/src/sqlancer/databend/ast/DatabendSelect.java new file mode 100644 index 000000000..adc75d1cc --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendSelect.java @@ -0,0 +1,41 @@ +package sqlancer.databend.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendToStringVisitor; + +public class DatabendSelect extends SelectBase + implements DatabendExpression, Select { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DatabendExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DatabendJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DatabendToStringVisitor.asString(this); + } +} diff --git a/src/sqlancer/databend/ast/DatabendTableReference.java b/src/sqlancer/databend/ast/DatabendTableReference.java new file mode 100644 index 000000000..7a98d3877 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendTableReference extends TableReferenceNode + implements DatabendExpression { + public DatabendTableReference(DatabendSchema.DatabendTable table) { + super(table); + } +} diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java new file mode 100644 index 000000000..119f93e5c --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -0,0 +1,87 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DatabendExpression { + + public DatabendUnaryPostfixOperation(DatabendExpression expr, DatabendUnaryPostfixOperator op) { + super(expr, op); + } + + public DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); + } + + public DatabendUnaryPostfixOperator getOp() { + return (DatabendUnaryPostfixOperator) op; + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } + + public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public DatabendDataType[] getInputDataTypes() { + return DatabendDataType.values(); + } + + @Override + public DatabendConstant apply(DatabendConstant value) { + return DatabendConstant.createBooleanConstant(value.isNull()); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public DatabendDataType[] getInputDataTypes() { + return DatabendDataType.values(); + } + + @Override + public DatabendConstant apply(DatabendConstant value) { + return DatabendConstant.createBooleanConstant(!value.isNull()); + } + }; + // IS + + private final String textRepresentations; + + DatabendUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static DatabendUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract DatabendDataType[] getInputDataTypes(); + + public abstract DatabendConstant apply(DatabendConstant value); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java new file mode 100644 index 000000000..ea6b65f07 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -0,0 +1,112 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements DatabendExpression { + + public DatabendUnaryPrefixOperation(DatabendExpression expr, DatabendUnaryPrefixOperator op) { + super(expr, op); + } + + public DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); + } + + public DatabendUnaryPrefixOperator getOp() { + return (DatabendUnaryPrefixOperator) op; + } + + @Override + public DatabendDataType getExpectedType() { + return getOp().getExpressionType(getExpression()); + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } + + public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType(DatabendExpression expr) { + return DatabendDataType.BOOLEAN; + } + + @Override + protected DatabendConstant apply(DatabendConstant value) { + if (value.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(!value.cast(DatabendDataType.BOOLEAN).asBoolean()); + } + } + }, + + UNARY_PLUS("+", DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType(DatabendExpression expr) { + return expr.getExpectedType(); + } + + @Override + protected DatabendConstant apply(DatabendConstant value) { + return value; + } + }, + UNARY_MINUS("-", DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType(DatabendExpression expr) { + return expr.getExpectedType(); + } + + @Override + protected DatabendConstant apply(DatabendConstant value) { + if (value.isNull()) { + return DatabendConstant.createNullConstant(); + } + try { + if (value.isInt()) { + return DatabendConstant.createIntConstant(-value.asInt()); + } else if (value.isFloat()) { + return DatabendConstant.createFloatConstant(-value.asFloat()); + } else { + return null; + } + } catch (UnsupportedOperationException e) { + return null; + } + } + }; + + private String textRepresentation; + private DatabendDataType[] dataTypes; + + DatabendUnaryPrefixOperator(String textRepresentation, DatabendDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract DatabendDataType getExpressionType(DatabendExpression expr); + + public DatabendDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + protected abstract DatabendConstant apply(DatabendConstant value); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java new file mode 100644 index 000000000..22336fd72 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendToStringVisitor; + +public final class DatabendDeleteGenerator extends AbstractDeleteGenerator { + + private final DatabendGlobalState globalState; + + private DatabendDeleteGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(DatabendGlobalState globalState) { + return new DatabendDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendDeleteFromTable(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + if (Randomly.getBoolean()) { + appendWhereClause(DatabendToStringVisitor.asString( + new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); + DatabendErrors.addExpressionErrors(errors); + } + } + +} diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java new file mode 100644 index 000000000..13ee80e95 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.databend.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendToStringVisitor; + +public class DatabendInsertGenerator extends AbstractInsertGenerator { + + private final DatabendGlobalState globalState; + + public DatabendInsertGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + return new DatabendInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + buildInsertInto(table.getName(), columns); + DatabendErrors.addInsertErrors(errors); + } + + @Override + protected void insertValue(DatabendColumn column) { + // TODO: 等Databend实现DEFAULT关键字,暂时注入普通的value + // if (Randomly.getBooleanWithRatherLowProbability()) { + // sb.append("DEFAULT"); + // } else { + // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); + // } + String value = DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 + sb.append(value); + + } + +} diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java new file mode 100644 index 000000000..8e52713b8 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -0,0 +1,438 @@ +package sqlancer.databend.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.databend.DatabendBugs; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendRowValue; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendAggregateOperation; +import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; +import sqlancer.databend.ast.DatabendBetweenOperation; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; +import sqlancer.databend.ast.DatabendBinaryComparisonOperation; +import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; +import sqlancer.databend.ast.DatabendBinaryLogicalOperation; +import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendCastOperation; +import sqlancer.databend.ast.DatabendColumnReference; +import sqlancer.databend.ast.DatabendColumnValue; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendInOperation; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendLikeOperation; +import sqlancer.databend.ast.DatabendOrderByTerm; +import sqlancer.databend.ast.DatabendPostFixText; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; + +public class DatabendNewExpressionGenerator + extends TypedExpressionGenerator + implements NoRECGenerator, + TLPWhereGenerator { + + private final DatabendGlobalState globalState; + private List tables; + + private final int maxDepth; + private boolean allowAggregateFunctions; + private DatabendRowValue rowValue; + + private Set columnOfLeafNode; + + public DatabendNewExpressionGenerator setRowValue(DatabendRowValue rowValue) { + this.rowValue = rowValue; + return this; + } + + public void setColumnOfLeafNode(Set columnOfLeafNode) { + this.columnOfLeafNode = columnOfLeafNode; + } + + public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + @Override + public DatabendExpression generateLeafNode(DatabendDataType dataType) { + if (Randomly.getBoolean()) { + return generateConstant(dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(dataType); + } else { + return createColumnOfType(dataType); + } + } + } + + final List filterColumns(DatabendDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + private DatabendExpression createColumnOfType(DatabendDataType type) { + List columns = filterColumns(type); + DatabendColumn column = Randomly.fromList(columns); + DatabendConstant value = rowValue == null ? null : rowValue.getValues().get(column); + if (columnOfLeafNode != null) { + columnOfLeafNode.add(DatabendColumnValue.create(column, value)); + } + return DatabendColumnValue.create(column, value); + } + + public List generateOrderBy() { + List randomColumns = Randomly.subset(columns); + return randomColumns.stream().map( + c -> new DatabendOrderByTerm(new DatabendColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) + .collect(Collectors.toList()); + } + + @Override + protected DatabendExpression generateExpression(DatabendDataType type, int depth) { + if (Randomly.getBooleanWithRatherLowProbability() || depth >= maxDepth) { + return generateLeafNode(type); + } + + switch (type) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case FLOAT: + case VARCHAR: + case DATE: + case TIMESTAMP: + case NULL: + return generateConstant(type); + default: + throw new AssertionError(); + } + } + + public List generateExpressions(int nr, DatabendDataType type) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(type)); + } + return expressions; + } + + private enum IntExpression { + UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION + } + + private DatabendExpression generateIntExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } + IntExpression intExpression = Randomly.fromOptions(IntExpression.values()); + switch (intExpression) { + case UNARY_OPERATION: + return new DatabendUnaryPrefixOperation(generateExpression(DatabendDataType.INT, depth + 1), + Randomly.getBoolean() ? DatabendUnaryPrefixOperator.UNARY_PLUS + : DatabendUnaryPrefixOperator.UNARY_MINUS); + case BINARY_ARITHMETIC_OPERATION: + return new DatabendBinaryArithmeticOperation(generateExpression(DatabendDataType.INT, depth + 1), + generateExpression(DatabendDataType.INT, depth + 1), + Randomly.fromOptions(DatabendBinaryArithmeticOperator.values())); + default: + throw new AssertionError(); + } + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; + // SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; + } + + DatabendExpression generateBooleanExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (DatabendBugs.bug15570) { + validOptions.remove(BooleanExpression.LIKE); + validOptions.remove(BooleanExpression.IN_OPERATION); + validOptions.remove(BooleanExpression.BETWEEN); + validOptions.remove(BooleanExpression.BINARY_COMPARISON); + } + if (DatabendBugs.bug15572) { + validOptions.remove(BooleanExpression.NOT); + } + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + return getPostfix(depth + 1); + case NOT: + return getNOT(depth + 1); + case BETWEEN: // TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 + return getBetween(depth + 1); + case IN_OPERATION: + return getIn(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return getBinaryLogical(depth + 1, DatabendDataType.BOOLEAN); + case BINARY_COMPARISON: + return getComparison(depth + 1); + case LIKE: + return getLike(depth + 1, DatabendDataType.VARCHAR); + default: + throw new AssertionError(); + } + + } + + DatabendExpression getPostfix(int depth) { + DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); + return new DatabendUnaryPostfixOperation( + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp); + } + + DatabendExpression getNOT(int depth) { + DatabendUnaryPrefixOperator op = DatabendUnaryPrefixOperator.NOT; + return new DatabendUnaryPrefixOperation(generateExpression(op.getRandomInputDataTypes(), depth), op); + } + + DatabendExpression getBetween(int depth) { + // 跳过boolean + DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() + .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); + + return new DatabendBetweenOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + generateExpression(dataType, depth), Randomly.getBoolean()); + } + + DatabendExpression getIn(int depth) { + DatabendDataType dataType = Randomly.fromOptions(DatabendDataType.values()); + DatabendExpression leftExpr = generateExpression(dataType, depth); + List rightExprs = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + rightExprs.add(generateExpression(dataType, depth)); + } + return new DatabendInOperation(leftExpr, rightExprs, Randomly.getBoolean()); + } + + DatabendExpression getBinaryLogical(int depth, DatabendDataType dataType) { + DatabendExpression expr = generateExpression(dataType, depth); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + expr = new DatabendBinaryLogicalOperation(expr, generateExpression(DatabendDataType.BOOLEAN, depth), + DatabendBinaryLogicalOperator.getRandom()); + } + return expr; + } + + DatabendExpression getComparison(int depth) { + // 跳过boolean + DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() + .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); + DatabendExpression leftExpr = generateExpression(dataType, depth); + DatabendExpression rightExpr = generateExpression(dataType, depth); + return new DatabendBinaryComparisonOperation(leftExpr, rightExpr, + Randomly.fromOptions(DatabendBinaryComparisonOperator.values())); + } + + DatabendExpression getLike(int depth, DatabendDataType dataType) { + return new DatabendLikeOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); + } + + public DatabendExpression generateExpressionWithExpectedResult(DatabendDataType type) { + // DatabendNewExpressionGenerator gen = new + // DatabendNewExpressionGenerator(globalState).setColumns(columns); + // gen.setRowValue(rowValue); + DatabendExpression expr; + do { + expr = this.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + @Override + public DatabendExpression generatePredicate() { + return generateExpression(DatabendDataType.BOOLEAN); + } + + @Override + public DatabendExpression negatePredicate(DatabendExpression predicate) { + return new DatabendUnaryPrefixOperation(predicate, DatabendUnaryPrefixOperator.NOT); + } + + @Override + public DatabendExpression isNull(DatabendExpression predicate) { + return new DatabendUnaryPostfixOperation(predicate, DatabendUnaryPostfixOperator.IS_NULL); + } + + public DatabendExpression generateConstant(DatabendDataType type, boolean isNullable) { + if (isNullable && Randomly.getBooleanWithSmallProbability()) { + createConstant(DatabendDataType.NULL); + } + return createConstant(type); + } + + @Override + public DatabendExpression generateConstant(DatabendDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant.createNullConstant(); + } + return createConstant(type); + } + + public DatabendExpression createConstant(DatabendDataType type) { + Randomly r = globalState.getRandomly(); + switch (type) { + case INT: + // TODO 已支持数值型string转化但仍然不支持运算符计算，待添加 + return DatabendConstant.createIntConstant(r.getInteger()); + case BOOLEAN: + // TODO 已支持boolean型string转化但仍然不支持运算符计算，待添加 + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return DatabendConstant.createFloatConstant((float) r.getDouble()); + case VARCHAR: + return DatabendConstant.createStringConstant(r.getString()); + case NULL: + return DatabendConstant.createNullConstant(); + case DATE: + return DatabendConstant.createDateConstant(r.getInteger()); + case TIMESTAMP: + return DatabendConstant.createTimestampConstant(r.getInteger()); + default: + throw new AssertionError(type); + } + } + + @Override + protected DatabendExpression generateColumn(DatabendDataType type) { + return null; + } + + @Override + protected DatabendDataType getRandomType() { + return Randomly.fromOptions(DatabendDataType.values()); + } + + @Override + protected boolean canGenerateColumnOfType(DatabendDataType type) { + return false; + } + + public DatabendExpression generateArgsForAggregate(DatabendAggregateFunction aggregateFunction) { + return new DatabendAggregateOperation( + generateExpressions(aggregateFunction.getNrArgs(), aggregateFunction.getRandomType()), + aggregateFunction); + } + + public DatabendExpression generateAggregate() { + DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + public DatabendExpression generateHavingClause() { + allowAggregateFunctions = true; + DatabendExpression expression = generateExpression(DatabendDataType.BOOLEAN); + allowAggregateFunctions = false; + return expression; + } + + @Override + public DatabendNewExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DatabendExpression generateBooleanExpression() { + return generateExpression(DatabendDataType.BOOLEAN); + } + + @Override + public DatabendSelect generateSelect() { + return new DatabendSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + List joins = DatabendJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DatabendTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DatabendSelect select, DatabendExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + DatabendExpression aggr = new DatabendAggregateOperation( + List.of(new DatabendColumnReference(new DatabendColumn("*", + new DatabendCompositeDataType(DatabendDataType.INT, 0), false, false))), + DatabendAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + List allColumns = columns.stream().map((c) -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DatabendSelect select, DatabendExpression whereCondition) { + DatabendExpression asText = new DatabendPostFixText(new DatabendCastOperation( + new DatabendPostFixText(whereCondition, + " IS NOT NULL AND " + DatabendToStringVisitor.asString(whereCondition)), + new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); + select.setFetchColumns(List.of(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new DatabendColumnReference(new DatabendColumn("*", null, false, false))); + } + return columns.stream().map(c -> new DatabendColumnReference(c)).collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java new file mode 100644 index 000000000..875c5afee --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -0,0 +1,92 @@ +package sqlancer.databend.gen; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendColumnValue; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; + +public final class DatabendRandomQuerySynthesizer { + + private DatabendRandomQuerySynthesizer() { + } + + public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { + DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); + List targetColumns = targetTables.getColumns(); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(targetColumns); + // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); + List columns = new ArrayList<>(); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); + int freeColumns = targetColumns.size(); + for (int i = 0; i < nrColumns; i++) { + // if (allowAggregates && Randomly.getBoolean()) { + DatabendExpression column = null; + if (freeColumns > 0 && Randomly.getBoolean()) { + column = new DatabendColumnValue(targetColumns.get(freeColumns - 1), null); + freeColumns -= 1; + columnOfLeafNode.add((DatabendColumnValue) column); + } else { + column = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); + } + columns.add(column); + } + DatabendSelect select = new DatabendSelect(); + boolean isDistinct = Randomly.getBoolean(); + select.setDistinct(isDistinct); + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + List joins = DatabendJoin.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); + } + + List noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean() && !noExprColumns.isEmpty() && !isDistinct) { + select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); + // TODO (for SELECT DISTINCT, ORDER BY expressions must appear in select list) + // isDistinct + // 需要orderby输入每个select list，可以用数字代替比如：1,2,3... + } + + if (Randomly.getBoolean()) { // 可能产生新的column叶子结点 + select.setHavingClause(gen.generateHavingClause()); + } + + noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean() && !noExprColumns.isEmpty()) { + select.setGroupByExpressions(noExprColumns); + } + + if (Randomly.getBoolean()) { + select.setLimitClause( + DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause( + DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + + return select; + } + +} diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java new file mode 100644 index 000000000..2c3416538 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -0,0 +1,73 @@ +package sqlancer.databend.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; + +public class DatabendTableGenerator extends AbstractTableGenerator { + + private DatabendGlobalState globalState; + private TypedExpressionGenerator gen; + + public DatabendTableGenerator() { + this.canAffectSchema = true; + } + + public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { + DatabendErrors.addExpressionErrors(errors); + String tableName = globalState.getSchema().getFreeTableName(); + appendCreateTable(tableName); + List columns = getNewColumns(); + gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + appendColumnDefinitions(columns); + } + + @Override + protected void appendColumnDefinition(DatabendColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } else { + sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 + } + + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 + gen.generateConstant(column.getType().getPrimitiveDataType()))); + sb.append(")"); + } + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + DatabendCompositeDataType columnType = DatabendCompositeDataType.getRandomWithoutNull(); + String columnName = String.format("c%d%s", i, columnType.getPrimitiveDataType().toString()); + columns.add(new DatabendColumn(columnName, columnType, false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java new file mode 100644 index 000000000..75ad57239 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -0,0 +1,29 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendToStringVisitor; + +public final class DatabendViewGenerator { + + private DatabendViewGenerator() { + } + + public static SQLQueryAdapter generate(DatabendGlobalState globalState) { + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append(" AS "); + sb.append(DatabendToStringVisitor + .asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addGroupByErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..4a72454b5 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java @@ -0,0 +1,150 @@ +package sqlancer.databend.test; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExpectedValueVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendRowValue; +import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendColumnValue; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; + +public class DatabendPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public DatabendPivotedQuerySynthesisOracle(DatabendGlobalState globalState) { + super(globalState); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addInsertErrors(errors); + } + + @Override + protected Query getRectifiedQuery() throws Exception { + DatabendTables randomTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); + List columns = randomTables.getColumns(); + DatabendSelect selectStatement = new DatabendSelect(); + boolean isDistinct = Randomly.getBoolean(); + selectStatement.setDistinct(isDistinct); + pivotRow = randomTables.getRandomRowValue(globalState.getConnection()); + fetchColumns = columns; + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new DatabendColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + selectStatement.setFromList( + randomTables.getTables().stream().map(t -> new DatabendTableReference(t)).collect(Collectors.toList())); + DatabendExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + DatabendExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + DatabendExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + if (!isDistinct) { + List orderBys = gen.generateOrderBy(); + selectStatement.setOrderByClauses(orderBys); + } + return new SQLQueryAdapter(DatabendToStringVisitor.asString(selectStatement), errors); + } + + private DatabendExpression generateRectifiedExpression(List columns, DatabendRowValue pivotRow) { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + gen.setRowValue(pivotRow); + DatabendExpression expr = gen.generateExpressionWithExpectedResult(DatabendDataType.BOOLEAN); + DatabendExpression result = null; + if (expr.getExpectedValue().isNull()) { + result = new DatabendUnaryPostfixOperation(expr, + DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator.IS_NULL); + } else if (!expr.getExpectedValue().cast(DatabendDataType.BOOLEAN).asBoolean()) { + result = new DatabendUnaryPrefixOperation(expr, + DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); + sb.append(pivotRowQuery.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (DatabendColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL "); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).toString()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + private DatabendColumn getFetchValueAliasedColumn(DatabendColumn c) { + DatabendColumn aliasedColumn = new DatabendColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType(), false, false); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + @Override + protected String getExpectedValues(DatabendExpression expr) { + return DatabendExpectedValueVisitor.asExpectedValues(expr); + } + + private List generateGroupByClause(List columns, DatabendRowValue rowValue) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> new DatabendColumnValue(c, rowValue.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private DatabendExpression generateLimit() { + if (Randomly.getBoolean()) { + return DatabendConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private DatabendExpression generateOffset() { + if (Randomly.getBoolean()) { + return DatabendConstant.createIntConstant(0); + } else { + return null; + } + } + +} diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..ee3656413 --- /dev/null +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -0,0 +1,194 @@ +package sqlancer.databend.test.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendAggregateOperation; +import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; +import sqlancer.databend.ast.DatabendAlias; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; +import sqlancer.databend.ast.DatabendBinaryOperation; +import sqlancer.databend.ast.DatabendCastOperation; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendFunctionOperation; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; + +public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + List aggregateFunctions = new ArrayList<>( + List.of(DatabendAggregateFunction.MAX, DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, + DatabendAggregateFunction.COUNT, DatabendAggregateFunction.AVG + /* , DatabendAggregateFunction.STDDEV_POP */)); + DatabendAggregateFunction aggregateFunction = Randomly.fromList(aggregateFunctions); + DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen + .generateArgsForAggregate(aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add((DatabendAggregateOperation) gen.generateAggregate()); // TODO 更换成非聚合函数 + } + select.setFetchColumns(Arrays.asList(aggregate)); + // if (Randomly.getBooleanWithRatherLowProbability()) { + // select.setOrderByClauses(gen.generateOrderBys()); + // } + originalQuery = DatabendToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResult == null && secondResult != null + || firstResult != null && (!firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(DatabendSelect select, + DatabendFunctionOperation aggregate, List from) { + String metamorphicQuery; + DatabendExpression whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); + DatabendExpression negatedClause = new DatabendUnaryPrefixOperation(whereClause, + DatabendUnaryPrefixOperator.NOT); + DatabendExpression notNullClause = new DatabendUnaryPostfixOperation(whereClause, + DatabendUnaryPostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + DatabendSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + DatabendSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + DatabendSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += DatabendToStringVisitor.asString(leftSelect) + " UNION ALL " + + DatabendToStringVisitor.asString(middleSelect) + " UNION ALL " + + DatabendToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + String resultString = null; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + try { + resultString = result.getString(1); + } catch (Exception e) { + throw new IgnoreMeException(); // TODO 超过integer范围无法格式化异常，还未有解决方案 + } + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List mapped(DatabendFunctionOperation aggregate) { + DatabendCastOperation count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(Arrays.asList(aggregate)); + case AVG: + DatabendFunctionOperation sum = new DatabendFunctionOperation<>( + aggregate.getArgs(), DatabendAggregateFunction.SUM); + count = new DatabendCastOperation( + new DatabendFunctionOperation<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), + new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); + return aliasArgs(Arrays.asList(sum, count)); + case STDDEV_POP: + DatabendFunctionOperation sumSquared = new DatabendFunctionOperation<>( + Arrays.asList(new DatabendBinaryOperation(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DatabendBinaryArithmeticOperator.MULTIPLICATION)), + DatabendAggregateFunction.SUM); + count = new DatabendCastOperation( + new DatabendFunctionOperation<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), + new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); + DatabendFunctionOperation avg = new DatabendFunctionOperation<>( + aggregate.getArgs(), DatabendAggregateFunction.AVG); + return aliasArgs(Arrays.asList(sumSquared, count, avg)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (DatabendExpression expr : originalAggregateArgs) { + args.add(new DatabendAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(DatabendFunctionOperation aggregate) { + switch (aggregate.getFunc()) { + case STDDEV_POP: + return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; + case AVG: + return "SUM(agg0)/SUM(agg1)"; + case COUNT: + return DatabendAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private DatabendSelect getSelect(List aggregates, List from, + DatabendExpression whereClause, List joinList) { + DatabendSelect select = new DatabendSelect(); + select.setFetchColumns(aggregates); + select.setFromList(from); + select.setWhereClause(whereClause); + select.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + select.setGroupByExpressions(List.of(gen.generateConstant(DatabendDataType.INT))); // TODO + // 仍可加强 + } + return select; + } + +} diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java new file mode 100644 index 000000000..611164902 --- /dev/null +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -0,0 +1,94 @@ +package sqlancer.databend.test.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.DatabendBugs; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendColumnReference; +import sqlancer.databend.ast.DatabendColumnValue; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; + +public class DatabendQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + DatabendSchema s; + DatabendTables targetTables; + DatabendNewExpressionGenerator gen; + DatabendSelect select; + + List groupByExpression; + + public DatabendQueryPartitioningBase(DatabendGlobalState state) { + super(state); + DatabendErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyAndViewTables(); + List randomColumn = targetTables.getColumns(); + + gen = new DatabendNewExpressionGenerator(state).setColumns(targetTables.getColumns()); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); + initializeTernaryPredicateVariants(); + select = new DatabendSelect(); + columnOfLeafNode + .addAll(randomColumn.stream().map(c -> new DatabendColumnValue(c, null)).collect(Collectors.toList())); + groupByExpression = new ArrayList<>(columnOfLeafNode); + + select.setFetchColumns( + randomColumn.stream().map(c -> new DatabendColumnReference(c)).collect(Collectors.toList())); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + if (!DatabendBugs.bug9236) { + List joins = DatabendJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + } + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List generateFetchColumns() { + List columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new DatabendColumnReference(new DatabendColumn("*", null, false, false))); + } else { + columns = generateRandomColumns(); + } + return columns; + } + + List generateRandomColumns() { + List columns; + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); + return columns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..ce20ef214 --- /dev/null +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -0,0 +1,46 @@ +package sqlancer.databend.test.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendToStringVisitor; + +public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningDistinctTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + // TODO 后期可以使用and来进行扩展 + // select.setWhereClause(DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + select.setWhereClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + if (Randomly.getBoolean()) { + select.setDistinct(false); + } + select.setWhereClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..9d45291b3 --- /dev/null +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java @@ -0,0 +1,51 @@ +package sqlancer.databend.test.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendColumnReference; +import sqlancer.databend.ast.DatabendExpression; + +public class DatabendQueryPartitioningGroupByTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningGroupByTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(groupByExpression); + select.setWhereClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } + + @Override + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java new file mode 100644 index 000000000..33de8c896 --- /dev/null +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -0,0 +1,67 @@ +package sqlancer.databend.test.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; + +public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); + } + // boolean orderBy = Randomly.getBoolean(); + boolean orderBy = false; // 关闭order by + if (orderBy) { // TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select + // select.setOrderByClauses(gen.generateOrderBys()); + List constants = new ArrayList<>(); + constants.add(new DatabendConstant.DatabendIntConstant( + Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByClauses(constants); + } + select.setGroupByExpressions(groupByExpression); + select.setHavingClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } + + @Override + protected DatabendExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List generateFetchColumns() { + return Collections.singletonList(gen.generateHavingClause()); + } + +} diff --git a/src/sqlancer/datafusion/DataFusionErrors.java b/src/sqlancer/datafusion/DataFusionErrors.java new file mode 100644 index 000000000..4be35017e --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionErrors.java @@ -0,0 +1,54 @@ +package sqlancer.datafusion; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class DataFusionErrors { + private DataFusionErrors() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + /* + * During Oracle Checks, if ANY query returns one of the following error Then the current oracle check will be + * skipped. e.g.: NoREC Q1 -> throw an expected error NoREC Q2 -> succeed Since it's a known error, `SQLancer` will + * skip this check and don't report bug. + * + * Note now it's implemented this way for simplicity This way might cause false negative, because Q1 and Q2 should + * both succeed or both fail TODO(datafusion): ensure both succeed or both fail + */ + public static List getExpectedExecutionErrors() { + ArrayList errors = new ArrayList<>(); + /* + * Expected + */ + errors.add("Error building plan"); // Randomly generated SQL is not valid and caused palning error + errors.add("Error during planning"); + errors.add("Execution error"); + errors.add("Overflow happened"); + errors.add("overflow"); + errors.add("Unsupported data type"); + errors.add("Divide by zero"); + /* + * Known bugs + */ + errors.add("to type Int64"); // https://github.com/apache/datafusion/issues/11252 + errors.add("bitwise"); // https://github.com/apache/datafusion/issues/11260 + errors.add("NestedLoopJoinExec"); // https://github.com/apache/datafusion/issues/11269 + /* + * False positives + */ + errors.add("Physical plan does not support logical expression AggregateFunction"); // False positive: when aggr + // is generated in where + // clause + + return errors; + } + + public static void registerExpectedExecutionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExecutionErrors()); + } +} diff --git a/src/sqlancer/datafusion/DataFusionOptions.java b/src/sqlancer/datafusion/DataFusionOptions.java new file mode 100644 index 000000000..582b2f658 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionOptions.java @@ -0,0 +1,21 @@ +package sqlancer.datafusion; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(commandDescription = "DataFusion") +public class DataFusionOptions implements DBMSSpecificOptions { + @Parameter(names = "--debug-info", description = "Show debug messages related to DataFusion", arity = 0) + public boolean showDebugInfo; + + @Override + public List getTestOracleFactory() { + return Arrays.asList(DataFusionOracleFactory.NOREC, DataFusionOracleFactory.QUERY_PARTITIONING_WHERE); + } + +} diff --git a/src/sqlancer/datafusion/DataFusionOracleFactory.java b/src/sqlancer/datafusion/DataFusionOracleFactory.java new file mode 100644 index 000000000..a7a3f21e8 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionOracleFactory.java @@ -0,0 +1,34 @@ +package sqlancer.datafusion; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public enum DataFusionOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create( + DataFusionProvider.DataFusionGlobalState globalState) throws SQLException { + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + QUERY_PARTITIONING_WHERE { + @Override + public TestOracle create( + DataFusionProvider.DataFusionGlobalState globalState) throws SQLException { + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(DataFusionErrors.getExpectedExecutionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + } +} diff --git a/src/sqlancer/datafusion/DataFusionProvider.java b/src/sqlancer/datafusion/DataFusionProvider.java new file mode 100644 index 000000000..37328e4ba --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionProvider.java @@ -0,0 +1,134 @@ +package sqlancer.datafusion; + +import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.DML; +import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import static sqlancer.datafusion.DataFusionUtil.displayTables; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionUtil.DataFusionInstanceID; +import sqlancer.datafusion.DataFusionUtil.DataFusionLogger; +import sqlancer.datafusion.gen.DataFusionInsertGenerator; +import sqlancer.datafusion.gen.DataFusionTableGenerator; + +@AutoService(DatabaseProvider.class) +public class DataFusionProvider extends SQLProviderAdapter { + + public DataFusionProvider() { + super(DataFusionGlobalState.class, DataFusionOptions.class); + } + + @Override + public void generateDatabase(DataFusionGlobalState globalState) throws Exception { + int tableCount = Randomly.fromOptions(1, 2, 3, 4, 5, 6, 7); + for (int i = 0; i < tableCount; i++) { + SQLQueryAdapter queryCreateRandomTable = new DataFusionTableGenerator().getQuery(globalState); + queryCreateRandomTable.execute(globalState); + globalState.updateSchema(); + globalState.dfLogger.appendToLog(DML, queryCreateRandomTable.toString() + "\n"); + } + + // Now only `INSERT` DML is supported + // If more DMLs are added later, should use`StatementExecutor` instead + // (see DuckDB's implementation for reference) + + globalState.updateSchema(); + List allTables = globalState.getSchema().getDatabaseTables(); + List allTablesName = allTables.stream().map(t -> t.getName()).collect(Collectors.toList()); + if (allTablesName.isEmpty()) { + dfAssert(false, "Generate Database failed."); + } + + // Randomly insert some data into existing tables + for (DataFusionTable table : allTables) { + int nInsertQuery = globalState.getRandomly().getInteger(0, globalState.getOptions().getMaxNumberInserts()); + + for (int i = 0; i < nInsertQuery; i++) { + SQLQueryAdapter insertQuery = null; + try { + insertQuery = DataFusionInsertGenerator.getQuery(globalState, table); + } catch (IgnoreMeException e) { + // Only for special case: table has 0 column + continue; + } + + insertQuery.execute(globalState); + globalState.dfLogger.appendToLog(DML, insertQuery.toString() + "\n"); + } + } + + // TODO(datafusion) add `DataFUsionLogType.STATE` for this whole db state log + if (globalState.getDbmsSpecificOptions().showDebugInfo) { + System.out.println(displayTables(globalState, allTablesName)); + } + } + + @Override + public SQLConnection createDatabase(DataFusionGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().showDebugInfo) { + System.out.println("A new database get created!\n"); + } + Properties props = new Properties(); + props.setProperty("UseEncryption", "false"); + // must set 'user' and 'password' to trigger server 'do_handshake()' + props.setProperty("user", "foo"); + props.setProperty("password", "bar"); + props.setProperty("create", globalState.getDatabaseName()); // Hack: use this property to let DataFusion server + // clear the current context + String url = "jdbc:arrow-flight-sql://127.0.0.1:50051"; + Connection connection = DriverManager.getConnection(url, props); + + return new SQLConnection(connection); + } + + @Override + public String getDBMSName() { + return "datafusion"; + } + + // If run SQLancer with multiple thread + // Each thread's instance will have its own `DataFusionGlobalState` + // It will store global states including: + // JDBC connection to DataFusion server + // Logger for this thread + public static class DataFusionGlobalState extends SQLGlobalState { + public DataFusionLogger dfLogger; + DataFusionInstanceID id; + + public DataFusionGlobalState() { + // HACK: test will only run in spawned thread, not main thread + // this way redundant logger files won't be created + if (Thread.currentThread().getName().equals("main")) { + return; + } + + id = new DataFusionInstanceID(Thread.currentThread().getName()); + try { + dfLogger = new DataFusionLogger(this, id); + } catch (Exception e) { + throw new IgnoreMeException(); + } + } + + @Override + protected DataFusionSchema readSchema() throws SQLException { + return DataFusionSchema.fromConnection(getConnection(), getDatabaseName()); + } + } +} diff --git a/src/sqlancer/datafusion/DataFusionSchema.java b/src/sqlancer/datafusion/DataFusionSchema.java new file mode 100644 index 000000000..d02e80c30 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionSchema.java @@ -0,0 +1,195 @@ +package sqlancer.datafusion; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.ast.DataFusionConstant; +import sqlancer.datafusion.ast.DataFusionExpression; + +public class DataFusionSchema extends AbstractSchema { + + public DataFusionSchema(List databaseTables) { + super(databaseTables); + } + + // update existing tables in DB by query again + // (like `show tables;`) + public static DataFusionSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + boolean isView = matchesViewName(tableName); + DataFusionTable t = new DataFusionTable(tableName, databaseColumns, isView); + for (DataFusionColumn c : databaseColumns) { + c.setTable(t); + } + + databaseTables.add(t); + } + + return new DataFusionSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select table_name " + "from information_schema.tables " + + "where table_schema='public'" + "order by table_name; ")) { + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + String.format("select * from information_schema.columns where table_name = '%s';", tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + boolean isNullable = rs.getString("is_nullable").contentEquals("YES"); + + DataFusionColumn c = new DataFusionColumn(columnName, + DataFusionDataType.parseFromDataFusionCatalog(dataType), isNullable); + columns.add(c); + } + } + } + + return columns; + } + + /* + * When adding a new type: 1. Update all methods inside this enum 2. Update all `DataFusionBaseExpr`'s signature, if + * it can support new type (in `DataFusionBaseExprFactory.java` + * + * Types are 'SQL DataType' in DataFusion's documentation + * https://datafusion.apache.org/user-guide/sql/data_types.html + */ + public enum DataFusionDataType { + + BIGINT, DOUBLE, BOOLEAN, NULL; + + public static DataFusionDataType getRandomWithoutNull() { + DataFusionDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DataFusionDataType.NULL); + return dt; + } + + // How to parse type in DataFusion's catalog to `DataFusionDataType` + // As displayed in: + // create table t1(v1 int, v2 bigint); + // select table_name, column_name, data_type from information_schema.columns; + public static DataFusionDataType parseFromDataFusionCatalog(String typeString) { + switch (typeString) { + case "Int64": + return DataFusionDataType.BIGINT; + case "Float64": + return DataFusionDataType.DOUBLE; + case "Boolean": + return DataFusionDataType.BOOLEAN; + default: + dfAssert(false, "Unreachable. All branches should be eovered"); + } + + dfAssert(false, "Unreachable. All branches should be eovered"); + return null; + } + + // TODO(datafusion) lots of hack here, should build our own Randomly later + public DataFusionExpression getRandomConstant(DataFusionGlobalState state) { + if (Randomly.getBooleanWithSmallProbability()) { + return DataFusionConstant.createNullConstant(); + } + switch (this) { + case BIGINT: + return DataFusionConstant.createIntConstant(state.getRandomly().getInteger()); + case BOOLEAN: + return new DataFusionConstant.DataFusionBooleanConstant(Randomly.getBoolean()); + case DOUBLE: + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + Double randomDouble = state.getRandomly().getDouble(); // [0.0, 1.0); + Double scaledDouble = (randomDouble - 0.5) * 2 * Double.MAX_VALUE; + return new DataFusionConstant.DataFusionDoubleConstant(scaledDouble); + } + String doubleStr = Randomly.fromOptions("'NaN'::Double", "'+Inf'::Double", "'-Inf'::Double", "-0.0", + "+0.0"); + return new DataFusionConstant.DataFusionDoubleConstant(doubleStr); + } + + return new DataFusionConstant.DataFusionDoubleConstant(state.getRandomly().getDouble()); + case NULL: + return DataFusionConstant.createNullConstant(); + default: + dfAssert(false, "Unreachable. All branches should be eovered"); + } + + dfAssert(false, "Unreachable. All branches should be eovered"); + return DataFusionConstant.createNullConstant(); + } + } + + public static class DataFusionColumn extends AbstractTableColumn { + + private final boolean isNullable; + + public DataFusionColumn(String name, DataFusionDataType columnType, boolean isNullable) { + super(name, null, columnType); + this.isNullable = isNullable; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class DataFusionTable + extends AbstractRelationalTable { + + public DataFusionTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + public static List getAllColumns(List tables) { + return tables.stream().map(AbstractTable::getColumns).flatMap(List::stream).collect(Collectors.toList()); + } + + public static List getRandomColumns(List tables) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new DataFusionColumn("*", DataFusionDataType.NULL, true)); + } + + List allColumns = getAllColumns(tables); + + return Randomly.nonEmptySubset(allColumns); + } + } + +} diff --git a/src/sqlancer/datafusion/DataFusionToStringVisitor.java b/src/sqlancer/datafusion/DataFusionToStringVisitor.java new file mode 100644 index 000000000..7d0d1b1b6 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionToStringVisitor.java @@ -0,0 +1,98 @@ +package sqlancer.datafusion; + +import java.util.List; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.datafusion.ast.DataFusionConstant; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionJoin; +import sqlancer.datafusion.ast.DataFusionSelect; + +public class DataFusionToStringVisitor extends NewToStringVisitor { + + public static String asString(DataFusionExpression expr) { + DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + public static String asString(List exprs) { + DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); + visitor.visit(exprs); + return visitor.get(); + } + + @Override + public void visitSpecific(DataFusionExpression expr) { + if (expr instanceof DataFusionConstant) { + visit((DataFusionConstant) expr); + } else if (expr instanceof DataFusionSelect) { + visit((DataFusionSelect) expr); + } else if (expr instanceof DataFusionJoin) { + visit((DataFusionJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DataFusionJoin join) { + visit((DataFusionExpression) join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + + sb.append(" JOIN "); + visit((DataFusionExpression) join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DataFusionConstant constant) { + sb.append(constant.toString()); + } + + private void visit(DataFusionSelect select) { + sb.append("SELECT "); + if (select.fetchColumnsString.isPresent()) { + sb.append(select.fetchColumnsString.get()); + } else { + visit(select.getFetchColumns()); + } + + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + +} diff --git a/src/sqlancer/datafusion/DataFusionUtil.java b/src/sqlancer/datafusion/DataFusionUtil.java new file mode 100644 index 000000000..8761bec9b --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionUtil.java @@ -0,0 +1,190 @@ +package sqlancer.datafusion; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; + +public final class DataFusionUtil { + private DataFusionUtil() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + // Display tables in `fromTableNames` + public static String displayTables(DataFusionGlobalState state, List fromTableNames) { + StringBuilder resultStringBuilder = new StringBuilder(); + for (String tableName : fromTableNames) { + String query = String.format("select * from %s", tableName); + try (Statement stat = state.getConnection().createStatement(); + ResultSet wholeTable = stat.executeQuery(query)) { + + ResultSetMetaData metaData = wholeTable.getMetaData(); + int columnCount = metaData.getColumnCount(); + + resultStringBuilder.append("Table: ").append(tableName).append("\n"); + for (int i = 1; i <= columnCount; i++) { + resultStringBuilder.append(metaData.getColumnName(i)).append(" (") + .append(metaData.getColumnTypeName(i)).append(")"); + if (i < columnCount) { + resultStringBuilder.append(", "); + } + } + resultStringBuilder.append("\n"); + + while (wholeTable.next()) { + for (int i = 1; i <= columnCount; i++) { + resultStringBuilder.append(wholeTable.getString(i)); + if (i < columnCount) { + resultStringBuilder.append(", "); + } + } + resultStringBuilder.append("\n"); + } + resultStringBuilder.append("----------------------------------------\n\n"); + + } catch (SQLException err) { + resultStringBuilder.append("Table: ").append(tableName).append("\n"); + resultStringBuilder.append("----------------------------------------\n\n"); + // resultStringBuilder.append("Error retrieving data from table ").append(tableName).append(": + // ").append(err.getMessage()).append("\n"); + } + } + + return resultStringBuilder.toString(); + } + + // During development, you might want to manually let this function call exit(1) to fail fast + public static void dfAssert(boolean condition, String message) { + if (!condition) { + // // Development mode assertion failure + // String methodName = Thread.currentThread().getStackTrace()[2]// .getMethodName(); + // System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); + // exit(1); + + throw new AssertionError(message); + } + } + + /* + * Fetch all DMLs from logs/database*-cur.log + */ + public static String getReplay(String dbname) { + String path = "./logs/datafusion/" + dbname + "-cur.log"; + String absolutePath = Paths.get(path).toAbsolutePath().toString(); + + StringBuilder reproducer = new StringBuilder(); + + try (BufferedReader reader = new BufferedReader(new FileReader(absolutePath))) { + String line; + while ((line = reader.readLine()) != null) { + // Check if the line contains the /*DML*/ marker + if (line.contains("/*DML*/")) { + reproducer.append(line).append("\n"); + } + } + } catch (IOException e) { + System.err.println("Error reading from file: " + e.getMessage()); + } + + return reproducer.toString(); + } + + // UID for different fuzzer runs + public static class DataFusionInstanceID { + private final String id; + + public DataFusionInstanceID(String dfID) { + id = dfID; + } + + @Override + public String toString() { + return id; // Return the id field when toString is called + } + } + + /* + * Extra logs stored in 'logs/datafusion_custom_log/' In case re-run overwrite previous logs + */ + public static class DataFusionLogger { + private final DataFusionInstanceID dfID; + private final DataFusionGlobalState state; + /* + * Log file handles + */ + private final File errorLogFile; + + public DataFusionLogger(DataFusionGlobalState globalState, DataFusionInstanceID id) throws Exception { + this.state = globalState; + this.dfID = id; + + // Setup datafusion_custom_log folder + File baseDir = new File("logs/datafusion_custom_log/"); + if (!baseDir.exists() && !baseDir.mkdirs()) { + throw new IOException("Failed to create 'datafusion_custom_log' directory/"); + } + + // Setup error.log + errorLogFile = new File(baseDir, "error_report.log"); + errorLogFile.createNewFile(); + } + + // Caller is responsible for adding '\n' at the end of logContent + public void appendToLog(DataFusionLogType logType, String logContent) { + FileWriter logFileWriter = null; + + // Determine which log file to use based on the LogType + String logLineHeader = ""; + switch (logType) { + case ERROR: + try { + logFileWriter = new FileWriter(errorLogFile, true); + } catch (IOException e) { + dfAssert(false, "Failed to create FileWriter for errorLogFIle"); + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String formattedDateTime = LocalDateTime.now().format(formatter); + logLineHeader = "Run@" + formattedDateTime + " (" + dfID + ")\n"; + break; + case DML: + logFileWriter = state.getLogger().getCurrentFileWriter(); + logLineHeader = "/*DML*/"; + break; + case SELECT: + logFileWriter = state.getLogger().getCurrentFileWriter(); + break; + default: + dfAssert(false, "All branch should be covered"); + } + + // Append content to the appropriate log file + if (logFileWriter != null) { + try { + logFileWriter.write(logLineHeader); + logFileWriter.write(logContent); + logFileWriter.flush(); + } catch (IOException e) { + String err = "Failed to write to " + logType + " log: " + e.getMessage(); + dfAssert(false, err); + } + } else { + dfAssert(false, "appending to log failed"); + } + } + + public enum DataFusionLogType { + ERROR, DML, SELECT + } + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java b/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java new file mode 100644 index 000000000..e59676be8 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DataFusionBinaryOperation extends NewBinaryOperatorNode + implements DataFusionExpression { + public DataFusionBinaryOperation(DataFusionExpression left, DataFusionExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionColumnReference.java b/src/sqlancer/datafusion/ast/DataFusionColumnReference.java new file mode 100644 index 000000000..2391ef694 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.datafusion.DataFusionSchema; + +public class DataFusionColumnReference extends + ColumnReferenceNode implements DataFusionExpression { + public DataFusionColumnReference(DataFusionSchema.DataFusionColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionConstant.java b/src/sqlancer/datafusion/ast/DataFusionConstant.java new file mode 100644 index 000000000..90a997fcf --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionConstant.java @@ -0,0 +1,97 @@ +package sqlancer.datafusion.ast; + +public class DataFusionConstant implements DataFusionExpression { + + private DataFusionConstant() { + } + + public static DataFusionExpression createIntConstant(long val) { + return new DataFusionIntConstant(val); + } + + public static DataFusionExpression createNullConstant() { + return new DataFusionNullConstant(); + } + + public static class DataFusionNullConstant extends DataFusionConstant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class DataFusionIntConstant extends DataFusionConstant { + + private final long value; + + public DataFusionIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class DataFusionDoubleConstant extends DataFusionConstant { + + private final String valueStr; + + public DataFusionDoubleConstant(double value) { + if (value == Double.POSITIVE_INFINITY) { + valueStr = "'+Inf'::Double"; + } else if (value == Double.NEGATIVE_INFINITY) { + valueStr = "'-Inf'::Double"; + } else if (Double.isNaN(value)) { + valueStr = "'NaN'::Double"; + } else if (Double.compare(value, -0.0) == 0) { + valueStr = "-0.0"; + } else { + valueStr = String.valueOf(value); + } + } + + // Make it more convenient to construct special value like -0, NaN, etc. + public DataFusionDoubleConstant(String valueStr) { + this.valueStr = valueStr; + } + + @Override + public String toString() { + return valueStr; + } + + } + + public static class DataFusionBooleanConstant extends DataFusionConstant { + + private final boolean value; + + public DataFusionBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + if (value) { + return "true"; + } else { + return "false"; + } + } + + } + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionExpression.java b/src/sqlancer/datafusion/ast/DataFusionExpression.java new file mode 100644 index 000000000..eaf84ba1d --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionExpression.java @@ -0,0 +1,8 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; + +public interface DataFusionExpression extends Expression { + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionFunction.java b/src/sqlancer/datafusion/ast/DataFusionFunction.java new file mode 100644 index 000000000..130fc04bf --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionFunction.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DataFusionFunction extends NewFunctionNode implements DataFusionExpression { + public DataFusionFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionJoin.java b/src/sqlancer/datafusion/ast/DataFusionJoin.java new file mode 100644 index 000000000..05718c5ad --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionJoin.java @@ -0,0 +1,91 @@ +package sqlancer.datafusion.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +/* + NOT IMPLEMENTED YET + */ +public class DataFusionJoin + implements DataFusionExpression, Join { + + private final DataFusionTableReference leftTable; + private final DataFusionTableReference rightTable; + private final JoinType joinType; + private DataFusionExpression onCondition; + + public DataFusionJoin(DataFusionTableReference leftTable, DataFusionTableReference rightTable, JoinType joinType, + DataFusionExpression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public static List getJoins(List tableList, + DataFusionGlobalState globalState) { + // [t1_join_t2, t1_join_t3, ...] + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + DataFusionTableReference leftTable = tableList.remove(0); + DataFusionTableReference rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + // TODO(datafusion) this `joinGen` can generate super chaotic exprsions, maybe we should make it more like a + // normal join expression + DataFusionExpressionGenerator joinGen = new DataFusionExpressionGenerator(globalState).setColumns(columns); + switch (DataFusionJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(DataFusionJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DataFusionJoin createInnerJoin(DataFusionTableReference left, DataFusionTableReference right, + DataFusionExpression predicate) { + return new DataFusionJoin(left, right, JoinType.INNER, predicate); + } + + public DataFusionTableReference getLeftTable() { + return leftTable; + } + + public DataFusionTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public DataFusionExpression getOnCondition() { + return onCondition; + } + + public enum JoinType { + INNER; + // NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + @Override + public void setOnClause(DataFusionExpression onClause) { + onCondition = onClause; + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java new file mode 100644 index 000000000..75bbb26ae --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -0,0 +1,43 @@ +package sqlancer.datafusion.ast; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; + +public class DataFusionSelect extends SelectBase implements DataFusionExpression, + Select { + public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base + + /* + * If set fetch columns with string It will override `fetchColumns` in base class when + * `DataFusionToStringVisitor.asString()` is called + * + * This method can be helpful to mutate select in oracle checks: SELECT [expr] ... -> SELECT SUM[expr] + */ + public void setFetchColumnsString(String selectExpr) { + this.fetchColumnsString = Optional.of(selectExpr); + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DataFusionExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DataFusionJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DataFusionToStringVisitor.asString(this); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionTableReference.java b/src/sqlancer/datafusion/ast/DataFusionTableReference.java new file mode 100644 index 000000000..14445576a --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.datafusion.DataFusionSchema; + +public class DataFusionTableReference extends TableReferenceNode + implements DataFusionExpression { + public DataFusionTableReference(DataFusionSchema.DataFusionTable table) { + super(table); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java b/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java new file mode 100644 index 000000000..ba5629460 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class DataFusionUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DataFusionExpression { + public DataFusionUnaryPostfixOperation(DataFusionExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java b/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java new file mode 100644 index 000000000..7109a12c2 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class DataFusionUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements DataFusionExpression { + public DataFusionUnaryPrefixOperation(DataFusionExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java b/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java new file mode 100644 index 000000000..0be57486e --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java @@ -0,0 +1,258 @@ +package sqlancer.datafusion.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; + +/* + Notes for adding new `DataFusionBaseExpr` support: + + Expression ASTs are constructed with Node<> class, it can traverse expression and convert AST to String representation + `DataFusionBaseExpr` implements `Operator<>` class, which is a field inside `Node<>` class, it includes operator properties like number of arguments, signature, or is this operator prefix/suffix, etc. + + To add new base expr (scalar functions, operators like '<<', 'AND' are all base expr): + 1. Add an enum variant to `DataFusionBaseExprType` + 2. Update `DataFusionBaseExprFactory.java` + (If a function support different argument number, make a new entry for each one. e.g. round scalar function support round(3.14) / round(3.14, 1), so it should be enum FUNC_ROUND1, FUNC_ROUND2) + */ +public class DataFusionBaseExpr implements Operator { + public String name; + public int nArgs; // number of input arguments + public DataFusionBaseExprCategory exprType; + public List possibleReturnTypes; + public List argTypes; + public boolean isVariadic; // Function supports arbitrary number of arguments, if set to `true`, it will + // override `nArgs` + + // Primary constructor + DataFusionBaseExpr(String name, int nArgs, DataFusionBaseExprCategory exprCategory, + List possibleReturnTypes, List argTypes, boolean isVariadic) { + this.name = name; + this.nArgs = nArgs; + this.exprType = exprCategory; + this.possibleReturnTypes = possibleReturnTypes; + this.argTypes = argTypes; + this.isVariadic = isVariadic; + } + + // Overloaded constructor assuming 'isVariadic' is false + DataFusionBaseExpr(String name, int nArgs, DataFusionBaseExprCategory exprCategory, + List possibleReturnTypes, List argTypes) { + this(name, nArgs, exprCategory, possibleReturnTypes, argTypes, false); + } + + public static DataFusionBaseExpr createCommonNumericFuncSingleArg(String name) { + return new DataFusionBaseExpr(name, 1, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + public static DataFusionBaseExpr createCommonNumericAggrFuncSingleArg(String name) { + return new DataFusionBaseExpr(name, 1, DataFusionBaseExprCategory.AGGREGATE, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + public static DataFusionBaseExpr createCommonNumericFuncTwoArgs(String name) { + return new DataFusionBaseExpr(name, 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + @Override + public String getTextRepresentation() { + return name; + } + + @Override + public String toString() { + return name; + } + + /* + * Class/Enum for `DataFusionBaseExpr` fields + */ + // Used to construct `src.common.ast.*Node` + public enum DataFusionBaseExprCategory { + UNARY_PREFIX, UNARY_POSTFIX, BINARY, FUNC, AGGREGATE + } + + /* + * Operators reference: https://datafusion.apache.org/user-guide/sql/operators.html Scalar functions: + * https://datafusion.apache.org/user-guide/sql/scalar_functions.html + */ + public enum DataFusionBaseExprType { + // Null Operators + IS_NULL, // IS NULL + IS_NOT_NULL, // IS NOT NULL + + // Numeric Operators + ADD, // 1 + 1 + SUB, // 1 - 1 + MULTIPLICATION, // 2 * 3 + DIVISION, // 8 / 4 + MODULO, // 5 % 3 + + // Comparison Operators + EQUAL, // 1 = 1 + EQUAL2, // 1 == 1 + NOT_EQUAL, // 1 != 2 + LESS_THAN, // 3 < 4 + LESS_THAN_OR_EQUAL_TO, // 3 <= 3 + GREATER_THAN, // 6 > 5 + GREATER_THAN_OR_EQUAL_TO, // 5 >= 5 + + // Distinctness operators + IS_DISTINCT_FROM, // 0 IS DISTINCT FROM NULL + IS_NOT_DISTINCT_FROM, // NULL IS NOT DISTINCT FROM NULL + + /* + * // Regular expression match operators REGEX_MATCH, // 'datafusion' ~ '^datafusion(-cli)*' + * REGEX_CASE_INSENSITIVE_MATCH, // 'datafusion' ~* '^DATAFUSION(-cli)*' NOT_REGEX_MATCH, // 'datafusion' !~ + * '^DATAFUSION(-cli)*' NOT_REGEX_CASE_INSENSITIVE_MATCH, // 'datafusion' !~* '^DATAFUSION(-cli)+' + * + * // Like pattern match operators LIKE_MATCH, // 'datafusion' ~~ 'dat_f%n' CASE_INSENSITIVE_LIKE_MATCH, // + * 'datafusion' ~~* 'Dat_F%n' NOT_LIKE_MATCH, // 'datafusion' !~~ 'Dat_F%n' NOT_CASE_INSENSITIVE_LIKE_MATCH // + * 'datafusion' !~~* 'Dat%F_n' + */ + + // Logical Operators + AND, // true and true + OR, // true or false + + // Bitwise Operators + BITWISE_AND, // 5 & 3 + BITWISE_OR, // 5 | 3 + BITWISE_XOR, // 5 ^ 3 + BITWISE_SHIFT_RIGHT, // 5 >> 3 + BITWISE_SHIFT_LEFT, // 5 << 3 + + /* + * // Other operators STRING_CONCATENATION, // 'Hello, ' || 'DataFusion!' ARRAY_CONTAINS, // + * make_array(1,2,3) @> make_array(1,3) ARRAY_IS_CONTAINED_BY // make_array(1,3) <@ make_array(1,2,3) + */ + + // Unary Prefix Operators + NOT, // NOT true + PLUS, // +7 + MINUS, // -3 + + /* + * Scalar Functions + */ + + // Math Functions + FUNC_ABS, // abs(-10) + FUNC_ACOS, // acos(1) + FUNC_ACOSH, // acosh(10) + FUNC_ASIN, // asin(1) + FUNC_ASINH, // asinh(1) + FUNC_ATAN, // atan(1) + FUNC_ATANH, // atanh(0.5) + FUNC_ATAN2, // atan2(10, 10) + FUNC_CBRT, // cbrt(27) + FUNC_CEIL, // ceil(9.2) + FUNC_COS, // cos(π/3) + FUNC_COSH, // cosh(0) + FUNC_DEGREES, // degrees(π) + FUNC_EXP, // exp(1) + FUNC_FACTORIAL, // factorial(5) + FUNC_FLOOR, // floor(3.7) + FUNC_GCD, // gcd(8, 12) + FUNC_ISNAN, // isnan(NaN) + FUNC_ISZERO, // iszero(0.0) + FUNC_LCM, // lcm(5, 15) + FUNC_LN, // ln(1) + FUNC_LOG, // log(100) + FUNC_LOG_WITH_BASE, // log(10, 100) + FUNC_LOG10, // log10(100) + FUNC_LOG2, // log2(32) + FUNC_NANVL, // nanvl(NaN, 3) + FUNC_PI, // pi() + FUNC_POW, // pow(2, 3) + FUNC_POWER, // power(2, 3) + FUNC_RADIANS, // radians(180) + // FUNC_RANDOM, // random() disabled because it's non-deterministic + FUNC_ROUND, // round(3.14159) + FUNC_ROUND_WITH_DECIMAL, // round(3.14159, 2) + FUNC_SIGNUM, // signum(-10) + FUNC_SIN, // sin(π/2) + FUNC_SINH, // sinh(1) + FUNC_SQRT, // sqrt(16) + FUNC_TAN, // tan(π/4) + FUNC_TANH, // tanh(1) + FUNC_TRUNC, // trunc(3.14159) + FUNC_TRUNC_WITH_DECIMAL, // trunc(3.14159, 2) + + // Conditional Functions + FUNC_COALESCE, // coalesce(NULL, 'default value') + FUNC_NULLIF, // nullif('value', 'value') + FUNC_NVL, // nvl(NULL, 'default value') + FUNC_NVL2, // nvl2('not null', 'return if not null', 'return if null') + FUNC_IFNULL, // ifnull(NULL, 'default value') + + // String Functions + + // Time and Date Functions + + // Array Functions + + // Struct Functions + + // Hashing Functions + + // Other Functions + + // Aggregate Functions + AGGR_MIN, AGGR_MAX, AGGR_SUM, AGGR_AVG, AGGR_COUNT, + } + + /* + * Because expressions are constructed in a top-down way, we have to infer argument type given return type. For each + * arg, if its corresponding element is `SameAsReturnType`, it should be the same as the type of expression's + * evaluated value. Else, it should be specific `DataFusionDataType` + * + * e.g. let's say we're generating a round(num, digit) of double type, its `argTypes` is: Arrays.asList( new + * ArgumentType.SameAsReturnType(), // First arg type as return type new ArgumentType.Fixed(new + * ArrayList<>(Array.asList(DataFusionDataType.INT)) // Second arg always Integer ) it means: its first argument + * should be the same as returned type (double), and the second arg should always be Int. + * + * Random expression generator's policy: SameAsReturnType -> generate an expr with the same type as its return type + * SameAsReturnType -> generate an expr with the same type as its 1st arg type Fixed(type1, type2, ... typeN) -> + * randomly choose a possible type (It will also generate completely random type/null ~10%) + * + * Note this defination is not comprehensive for native `DataFusion` types. It's just for simplicity and should + * cover most common cases + */ + public abstract static class ArgumentType { + private ArgumentType() { + } + + public static class SameAsReturnType extends ArgumentType { + } + + public static class SameAsFirstArgType extends ArgumentType { + } + + public static class Fixed extends ArgumentType { + public List fixedType; // It's a list to support different possible arg types. + + public Fixed(List fixedType) { + this.fixedType = fixedType; + } + + public List getType() { + return fixedType; + } + } + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java b/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java new file mode 100644 index 000000000..d3fe39972 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java @@ -0,0 +1,391 @@ +package sqlancer.datafusion.gen; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericAggrFuncSingleArg; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericFuncSingleArg; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericFuncTwoArgs; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprCategory; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; + +public final class DataFusionBaseExprFactory { + private DataFusionBaseExprFactory() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + public static DataFusionBaseExpr createExpr(DataFusionBaseExprType type) { + switch (type) { + case IS_NULL: + return new DataFusionBaseExpr("IS NULL", 1, DataFusionBaseExprCategory.UNARY_POSTFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.DOUBLE, DataFusionDataType.BIGINT, DataFusionDataType.NULL))))); + case IS_NOT_NULL: + return new DataFusionBaseExpr("IS NOT NULL", 1, DataFusionBaseExprCategory.UNARY_POSTFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.DOUBLE, DataFusionDataType.BIGINT, DataFusionDataType.NULL))))); + case BITWISE_AND: + return new DataFusionBaseExpr("&", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_OR: + return new DataFusionBaseExpr("|", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_XOR: + return new DataFusionBaseExpr("^", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_SHIFT_RIGHT: + return new DataFusionBaseExpr(">>", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case BITWISE_SHIFT_LEFT: + return new DataFusionBaseExpr("<<", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case NOT: + return new DataFusionBaseExpr("NOT", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))))); + case PLUS: // unary prefix '+' + return new DataFusionBaseExpr("+", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MINUS: // unary prefix '-' + return new DataFusionBaseExpr("-", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MULTIPLICATION: + return new DataFusionBaseExpr("*", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case DIVISION: + return new DataFusionBaseExpr("/", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MODULO: + return new DataFusionBaseExpr("%", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case EQUAL: + return new DataFusionBaseExpr("=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case EQUAL2: + return new DataFusionBaseExpr("==", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case NOT_EQUAL: + return new DataFusionBaseExpr("!=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case LESS_THAN: + return new DataFusionBaseExpr("<", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case LESS_THAN_OR_EQUAL_TO: + return new DataFusionBaseExpr("<=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case GREATER_THAN: + return new DataFusionBaseExpr(">", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case GREATER_THAN_OR_EQUAL_TO: + return new DataFusionBaseExpr(">=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case IS_DISTINCT_FROM: + return new DataFusionBaseExpr("IS DISTINCT FROM", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case IS_NOT_DISTINCT_FROM: + return new DataFusionBaseExpr("IS NOT DISTINCT FROM", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case AND: + return new DataFusionBaseExpr("AND", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))) // arg2 + )); + case OR: + return new DataFusionBaseExpr("OR", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))) // arg2 + )); + case ADD: // binary arithmetic operator '+' + return new DataFusionBaseExpr("+", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))) // arg2 + )); + case SUB: // binary arithmetic operator '-' + return new DataFusionBaseExpr("-", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), // arg1 + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))) // arg2 + )); + case FUNC_ABS: + return createCommonNumericFuncSingleArg("ABS"); + case FUNC_ACOS: + return createCommonNumericFuncSingleArg("ACOS"); + case FUNC_ACOSH: + return createCommonNumericFuncSingleArg("ACOSH"); + case FUNC_ASIN: + return createCommonNumericFuncSingleArg("ASIN"); + case FUNC_ASINH: + return createCommonNumericFuncSingleArg("ASINH"); + case FUNC_ATAN: + return createCommonNumericFuncSingleArg("ATAN"); + case FUNC_ATANH: + return createCommonNumericFuncSingleArg("ATANH"); + case FUNC_ATAN2: + return createCommonNumericFuncTwoArgs("ATAN2"); + case FUNC_CBRT: + return createCommonNumericFuncSingleArg("CBRT"); + case FUNC_CEIL: + return createCommonNumericFuncSingleArg("CEIL"); + case FUNC_COS: + return createCommonNumericFuncSingleArg("COS"); + case FUNC_COSH: + return createCommonNumericFuncSingleArg("COSH"); + case FUNC_DEGREES: + return createCommonNumericFuncSingleArg("DEGREES"); + case FUNC_EXP: + return createCommonNumericFuncSingleArg("EXP"); + case FUNC_FACTORIAL: + return createCommonNumericFuncSingleArg("FACTORIAL"); + case FUNC_FLOOR: + return createCommonNumericFuncSingleArg("FLOOR"); + case FUNC_GCD: + return new DataFusionBaseExpr("GCD", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_ISNAN: + return createCommonNumericFuncSingleArg("ISNAN"); + case FUNC_ISZERO: + return createCommonNumericFuncSingleArg("ISZERO"); + case FUNC_LCM: + return createCommonNumericFuncTwoArgs("LCM"); + case FUNC_LN: + return createCommonNumericFuncSingleArg("LN"); + case FUNC_LOG: + return createCommonNumericFuncSingleArg("LOG"); + case FUNC_LOG_WITH_BASE: + return createCommonNumericFuncTwoArgs("LOG"); + case FUNC_LOG10: + return createCommonNumericFuncSingleArg("LOG10"); + case FUNC_LOG2: + return createCommonNumericFuncSingleArg("LOG2"); + case FUNC_NANVL: + return createCommonNumericFuncTwoArgs("NANVL"); + case FUNC_PI: + return new DataFusionBaseExpr("PI", 0, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), Arrays.asList()); + case FUNC_POW: + return createCommonNumericFuncSingleArg("POW"); + case FUNC_POWER: + return createCommonNumericFuncSingleArg("POWER"); + case FUNC_RADIANS: + return createCommonNumericFuncSingleArg("RADIANS"); + case FUNC_ROUND: + return createCommonNumericFuncSingleArg("ROUND"); + case FUNC_ROUND_WITH_DECIMAL: + return new DataFusionBaseExpr("ROUND", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case FUNC_SIGNUM: + return createCommonNumericFuncSingleArg("SIGNUM"); + case FUNC_SIN: + return createCommonNumericFuncSingleArg("SIN"); + case FUNC_SINH: + return createCommonNumericFuncSingleArg("SINH"); + case FUNC_SQRT: + return createCommonNumericFuncSingleArg("SQRT"); + case FUNC_TAN: + return createCommonNumericFuncSingleArg("TAN"); + case FUNC_TANH: + return createCommonNumericFuncSingleArg("TANH"); + case FUNC_TRUNC: + return createCommonNumericFuncSingleArg("TRUNC"); + case FUNC_TRUNC_WITH_DECIMAL: + return new DataFusionBaseExpr("TRUNC", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case FUNC_COALESCE: + return new DataFusionBaseExpr("COALESCE", -1, // overide by variadic + DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), Arrays.asList(), true); + case FUNC_NULLIF: + return new DataFusionBaseExpr("NULLIF", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_NVL: + return new DataFusionBaseExpr("NVL", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_NVL2: + return new DataFusionBaseExpr("NVL2", 3, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_IFNULL: + return new DataFusionBaseExpr("IFNULL", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + + case AGGR_MIN: + return createCommonNumericAggrFuncSingleArg("MIN"); + case AGGR_MAX: + return createCommonNumericAggrFuncSingleArg("MAX"); + case AGGR_AVG: + return createCommonNumericAggrFuncSingleArg("AVG"); + case AGGR_SUM: + return createCommonNumericAggrFuncSingleArg("SUM"); + case AGGR_COUNT: + return new DataFusionBaseExpr("COUNT", -1, DataFusionBaseExprCategory.AGGREGATE, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE)))), + true); + default: + dfAssert(false, "Unreachable. Unimplemented branch for type " + type); + } + + dfAssert(false, "Unreachable. Unimplemented branch for type " + type); + return null; + } + + // if input is Optional.empty(), return all possible `DataFusionBaseExpr`s + // else, return all `DataFusionBaseExpr` which might be evaluated to arg's type + public static List getExprsWithReturnType(Optional dataTypeOptional) { + List allExpressions = Arrays.stream(DataFusionBaseExprType.values()) + .map(DataFusionBaseExprFactory::createExpr).collect(Collectors.toList()); + + if (!dataTypeOptional.isPresent()) { + return allExpressions; // If Optional is empty, return all expressions + } + + DataFusionDataType filterType = dataTypeOptional.get(); + List exprsWithReturnType = allExpressions.stream() + .filter(expr -> expr.possibleReturnTypes.contains(filterType)).collect(Collectors.toList()); + + if (Randomly.getBoolean()) { + // Too many similar function, so test them less often + return exprsWithReturnType; + } + + return exprsWithReturnType.stream().filter(expr -> expr.exprType != DataFusionBaseExprCategory.FUNC) + .collect(Collectors.toList()); + } + + public static DataFusionBaseExpr getRandomAggregateExpr() { + List allAggrExpressions = Arrays.stream(DataFusionBaseExprType.values()) + .map(DataFusionBaseExprFactory::createExpr) + .filter(expr -> expr.exprType == DataFusionBaseExprCategory.AGGREGATE).collect(Collectors.toList()); + + return Randomly.fromList(allAggrExpressions); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java new file mode 100644 index 000000000..520043fa1 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -0,0 +1,305 @@ +package sqlancer.datafusion.gen; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import static sqlancer.datafusion.gen.DataFusionBaseExprFactory.createExpr; +import static sqlancer.datafusion.gen.DataFusionBaseExprFactory.getExprsWithReturnType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; +import sqlancer.datafusion.ast.DataFusionBinaryOperation; +import sqlancer.datafusion.ast.DataFusionColumnReference; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionFunction; +import sqlancer.datafusion.ast.DataFusionJoin; +import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.ast.DataFusionTableReference; +import sqlancer.datafusion.ast.DataFusionUnaryPostfixOperation; +import sqlancer.datafusion.ast.DataFusionUnaryPrefixOperation; +import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; + +public final class DataFusionExpressionGenerator + extends TypedExpressionGenerator implements + NoRECGenerator, + TLPWhereGenerator { + + private List tables; + private final DataFusionGlobalState globalState; + + public DataFusionExpressionGenerator(DataFusionGlobalState globalState) { + this.globalState = globalState; + } + + @Override + protected DataFusionDataType getRandomType() { + DataFusionDataType dt; + do { + dt = Randomly.fromOptions(DataFusionDataType.values()); + } while (dt == DataFusionDataType.NULL); + + return dt; + } + + @Override + protected boolean canGenerateColumnOfType(DataFusionDataType type) { + return true; + } + + @Override + protected DataFusionExpression generateExpression(DataFusionDataType type, int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + DataFusionDataType expectedType = type; + if (Randomly.getBooleanWithRatherLowProbability()) { // ~10% + expectedType = DataFusionDataType.getRandomWithoutNull(); + } + return generateLeafNode(expectedType); + } + + // nested aggregate is not allowed, so occasionally apply it + Boolean includeAggr = Randomly.getBooleanWithSmallProbability(); + List possibleBaseExprs = getExprsWithReturnType(Optional.of(type)).stream() + // Conditinally apply filter if `includeAggr` set to false + .filter(expr -> includeAggr || expr.exprType != DataFusionBaseExpr.DataFusionBaseExprCategory.AGGREGATE) + .collect(Collectors.toList()); + + if (possibleBaseExprs.isEmpty()) { + dfAssert(type == DataFusionDataType.NULL, "should able to generate expression with type " + type); + return generateLeafNode(type); + } + + DataFusionBaseExpr randomExpr = Randomly.fromList(possibleBaseExprs); + switch (randomExpr.exprType) { + case UNARY_PREFIX: + DataFusionDataType argType = null; + dfAssert(randomExpr.argTypes.size() == 1 && randomExpr.nArgs == 1, + "Unary expression should only have 1 argument" + randomExpr.argTypes); + if (randomExpr.argTypes.get(0) instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + argType = Randomly.fromList(possibleArgTypes.fixedType); + } else { + argType = type; + } + + return new DataFusionUnaryPrefixOperation(generateExpression(argType, depth + 1), randomExpr); + case UNARY_POSTFIX: + dfAssert(randomExpr.argTypes.size() == 1 && randomExpr.nArgs == 1, + "Unary expression should only have 1 argument" + randomExpr.argTypes); + if (randomExpr.argTypes.get(0) instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + argType = Randomly.fromList(possibleArgTypes.fixedType); + } else { + argType = type; + } + + return new DataFusionUnaryPostfixOperation(generateExpression(argType, depth + 1), randomExpr); + case BINARY: + dfAssert(randomExpr.argTypes.size() == 2 && randomExpr.nArgs == 2, + "Binrary expression should only have 2 argument" + randomExpr.argTypes); + List argTypeList = new ArrayList<>(); // types of current expression's input + // arguments + for (ArgumentType argumentType : randomExpr.argTypes) { + if (argumentType instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + dfAssert(!possibleArgTypes.fixedType.isEmpty(), "possible types can't be an empty list"); + DataFusionDataType determinedType = Randomly.fromList(possibleArgTypes.fixedType); + argTypeList.add(determinedType); + } else if (argumentType instanceof ArgumentType.SameAsFirstArgType) { + dfAssert(!argTypeList.isEmpty(), "First argument can't have argument type `SameAsFirstArgType`"); + DataFusionDataType firstArgType = argTypeList.get(0); + argTypeList.add(firstArgType); + } else { + // Same as expression return type + argTypeList.add(type); + } + } + + return new DataFusionBinaryOperation(generateExpression(argTypeList.get(0), depth + 1), + generateExpression(argTypeList.get(1), depth + 1), randomExpr); + case AGGREGATE: + // Fall through + case FUNC: + return generateFunctionExpression(type, depth, randomExpr); + default: + dfAssert(false, "unreachable"); + } + + dfAssert(false, "unreachable"); + return null; + } + + public DataFusionExpression generateFunctionExpression(DataFusionDataType type, int depth, + DataFusionBaseExpr exprType) { + if (exprType.isVariadic || Randomly.getBooleanWithSmallProbability()) { + // TODO(datafusion) maybe add possible types. e.g. some function have signature + // variadic(INT/DOUBLE), then + // only randomly pick from INT and DOUBLE + int nArgs = Randomly.smallNumber(); // 0, 2, 4, ... smaller one is more likely + return new DataFusionFunction(generateExpressions(nArgs), exprType); + } + + List funcArgTypeList = new ArrayList<>(); // types of current expression's input arguments + int i = 0; + for (ArgumentType argumentType : exprType.argTypes) { + if (argumentType instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) exprType.argTypes.get(i); + dfAssert(!possibleArgTypes.fixedType.isEmpty(), "possible types can't be an empty list"); + DataFusionDataType determinedType = Randomly.fromList(possibleArgTypes.fixedType); + funcArgTypeList.add(determinedType); + } else if (argumentType instanceof ArgumentType.SameAsFirstArgType) { + dfAssert(!funcArgTypeList.isEmpty(), "First argument can't have argument type `SameAsFirstArgType`"); + DataFusionDataType firstArgType = funcArgTypeList.get(0); + funcArgTypeList.add(firstArgType); + } else { + // Same as expression return type + funcArgTypeList.add(type); + } + i++; + } + + List argExpressions = new ArrayList<>(); + + for (DataFusionDataType dataType : funcArgTypeList) { + argExpressions.add(generateExpression(dataType, depth + 1)); + } + + return new DataFusionFunction(argExpressions, exprType); + } + + List filterColumns(DataFusionDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + @Override + protected DataFusionExpression generateColumn(DataFusionDataType type) { + // HACK: if no col of such type exist, generate constant value instead + List colsOfType = filterColumns(type); + if (colsOfType.isEmpty()) { + return generateConstant(type); + } + + DataFusionColumn column = Randomly.fromList(colsOfType); + return new DataFusionColumnReference(column); + } + + @Override + public DataFusionExpression generateConstant(DataFusionDataType type) { + return type.getRandomConstant(globalState); + } + + @Override + public DataFusionExpression generatePredicate() { + return generateExpression(DataFusionDataType.BOOLEAN, 0); + } + + @Override + public DataFusionExpression negatePredicate(DataFusionExpression predicate) { + return new DataFusionUnaryPrefixOperation(predicate, createExpr(DataFusionBaseExprType.NOT)); + } + + @Override + public DataFusionExpression isNull(DataFusionExpression expr) { + return new DataFusionUnaryPostfixOperation(expr, createExpr(DataFusionBaseExprType.IS_NULL)); + } + + public static class DataFusionCastOperation extends NewUnaryPostfixOperatorNode { + + public DataFusionCastOperation(DataFusionExpression expr, DataFusionDataType type) { + super(expr, new Operator() { + + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + + } + + @Override + public DataFusionExpressionGenerator setTablesAndColumns(AbstractTables tables) { + List randomTables = Randomly.nonEmptySubset(tables.getTables()); + int maxSize = Randomly.fromOptions(1, 2, 3, 4); + if (randomTables.size() > maxSize) { + randomTables = randomTables.subList(0, maxSize); + } + this.columns = DataFusionTable.getAllColumns(randomTables); + this.tables = randomTables; + + return this; + } + + @Override + public DataFusionExpression generateBooleanExpression() { + return generateExpression(DataFusionDataType.BOOLEAN); + } + + @Override + public DataFusionSelect generateSelect() { + return new DataFusionSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DataFusionTableReference(t)) + .collect(Collectors.toList()); + List joins = DataFusionJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DataFusionTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DataFusionSelect select, DataFusionExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + select.setFetchColumnsString("COUNT(*)"); + } else { + List allColumns = columns.stream().map((c) -> new DataFusionColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DataFusionSelect select, DataFusionExpression whereCondition) { + String fetchColumn = String.format("COUNT(CASE WHEN %S THEN 1 ELSE NULL END)", + DataFusionToStringVisitor.asString(whereCondition)); + select.setFetchColumnsString(fetchColumn); + select.setWhereClause(null); + + return select.asString(); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + List randomColumns = DataFusionTable.getRandomColumns(tables); + return randomColumns.stream().map((c) -> new DataFusionColumnReference(c)).collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java new file mode 100644 index 000000000..36a178791 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -0,0 +1,43 @@ +package sqlancer.datafusion.gen; + +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; + +public class DataFusionInsertGenerator extends AbstractInsertGenerator { + + private final DataFusionGlobalState globalState; + private final DataFusionTable targetTable; + + public DataFusionInsertGenerator(DataFusionGlobalState globalState, DataFusionTable targetTable) { + this.globalState = globalState; + this.targetTable = targetTable; + } + + public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFusionTable targetTable) { + return new DataFusionInsertGenerator(globalState, targetTable).getStatement(); + } + + @Override + public void buildStatement() { + if (targetTable.getColumns().isEmpty()) { + throw new IgnoreMeException(); + } + List columns = targetTable.getRandomNonEmptyColumnSubset(); + buildInsertInto(targetTable.getName(), columns); + } + + @Override + protected void insertValue(DataFusionColumn col) { + String val = DataFusionToStringVisitor + .asString(new DataFusionExpressionGenerator(globalState).generateConstant(col.getType())); + sb.append(val); + } + +} diff --git a/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java b/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java new file mode 100644 index 000000000..adececaa7 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.datafusion.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; + +public class DataFusionTableGenerator { + + // Randomly generate a query like 'create table t1 (v1 bigint, v2 boolean)' + public SQLQueryAdapter getQuery(DataFusionGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + + int colCount = Randomly.smallNumber() + 1 + (Randomly.getBoolean() ? 1 : 0); + for (int i = 0; i < colCount; i++) { + sb.append("v").append(i).append(" ").append(DataFusionDataType.getRandomWithoutNull().toString()); + + if (i != colCount - 1) { + sb.append(", "); + } + } + + sb.append(");"); + + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml new file mode 100644 index 000000000..332a88e30 --- /dev/null +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "datafusion-server" +edition = "2021" +description = "Standalone DataFusion server" +license = "Apache-2.0" + +[dependencies] +ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } +arrow = { version = "52.1.0", features = ["prettyprint"] } +arrow-array = { version = "52.1.0", default-features = false, features = ["chrono-tz"] } +arrow-buffer = { version = "52.1.0", default-features = false } +arrow-flight = { version = "52.1.0", features = ["flight-sql-experimental"] } +arrow-ipc = { version = "52.1.0", default-features = false, features = ["lz4"] } +arrow-ord = { version = "52.1.0", default-features = false } +arrow-schema = { version = "52.1.0", default-features = false } +arrow-string = { version = "52.1.0", default-features = false } +async-trait = "0.1.73" +bytes = "1.4" +chrono = { version = ">=0.4.34, <0.4.40", default-features = false } +dashmap = "5.5.0" +# This version is for SQLancer CI run +datafusion = { version = "40.0.0" } +# Use following line if you want to test against the latest main branch of DataFusion +# datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main" } +env_logger = "0.11" +futures = "0.3" +half = { version = "2.2.1", default-features = false } +hashbrown = { version = "0.14.5", features = ["raw"] } +log = "0.4" +num_cpus = "1.13.0" +object_store = { version = "0.10.1", default-features = false } +parking_lot = "0.12" +parquet = { version = "52.0.0", default-features = false, features = ["arrow", "async", "object_store"] } +rand = "0.8" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1" +tokio = { version = "1.36", features = ["macros", "rt", "sync"] } +tonic = "0.11" +uuid = "1.0" +prost = { version = "0.12", default-features = false } +prost-derive = { version = "0.12", default-features = false } +mimalloc = { version = "0.1", default-features = false } + +[[bin]] +name = "datafusion-server" +path = "src/main.rs" \ No newline at end of file diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs new file mode 100644 index 000000000..057c34883 --- /dev/null +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -0,0 +1,463 @@ +use arrow::array::{ArrayRef, StringArray}; +use arrow::ipc::writer::IpcWriteOptions; +use arrow::record_batch::RecordBatch; +use arrow_flight::encode::FlightDataEncoderBuilder; +use arrow_flight::flight_descriptor::DescriptorType; +use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; +use arrow_flight::sql::server::{FlightSqlService, PeekableFlightDataStream}; +use arrow_flight::sql::{ + ActionClosePreparedStatementRequest, ActionCreatePreparedStatementRequest, + ActionCreatePreparedStatementResult, Any, CommandGetTables, CommandPreparedStatementQuery, + CommandPreparedStatementUpdate, ProstMessageExt, SqlInfo, +}; +use arrow_flight::{ + Action, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, + IpcMessage, SchemaAsIpc, Ticket, +}; +use arrow_schema::{DataType, Field, Schema}; +use dashmap::DashMap; +use datafusion::logical_expr::LogicalPlan; +use datafusion::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; +use futures::{Stream, StreamExt, TryStreamExt}; +use log::info; +use mimalloc::MiMalloc; +use prost::Message; +use std::pin::Pin; +use std::sync::Arc; +use tokio::sync::Mutex; +use tonic::metadata::MetadataValue; +use tonic::transport::Server; +use tonic::{Request, Response, Status, Streaming}; +use uuid::Uuid; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +macro_rules! status { + ($desc:expr, $err:expr) => { + Status::internal(format!("{}: {} at {}:{}", $desc, $err, file!(), line!())) + }; +} + +/// Adapted from https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/flight/flight_sql_server.rs +/// Can be used as a remote DataFusion server and connected by `JDBC` from client +/// Supported SQL statements: +/// CREATE +/// INSERT +/// SELECT +/// +/// Only single client is supported +/// For now use `ctx` instead of `contexts` inside `FlightSqlServiceImpl` +/// +/// === Below origianl comment === +/// +/// This example shows how to wrap DataFusion with `FlightSqlService` to support connecting +/// to a standalone DataFusion-based server with a JDBC client, using the open source "JDBC Driver +/// for Arrow Flight SQL". +/// +/// To install the JDBC driver in DBeaver for example, see these instructions: +/// https://docs.dremio.com/software/client-applications/dbeaver/ +/// When configuring the driver, specify property "UseEncryption" = false +/// +/// JDBC connection string: "jdbc:arrow-flight-sql://127.0.0.1:50051/" +/// +/// Based heavily on Ballista's implementation: https://github.com/apache/datafusion-ballista/blob/main/ballista/scheduler/src/flight_sql.rs +/// and the example in arrow-rs: https://github.com/apache/arrow-rs/blob/master/arrow-flight/examples/flight_sql_server.rs +/// +#[tokio::main] +async fn main() -> Result<(), Box> { + env_logger::init(); + let addr = "0.0.0.0:50051".parse()?; + let session_ctx = SessionContext::new_with_config( + SessionConfig::new().with_information_schema(true), // enable catalog + ); + let service = FlightSqlServiceImpl { + contexts: Default::default(), + statements: Default::default(), + results: Default::default(), + ctx: Arc::new(Mutex::new(session_ctx)), + }; + info!("Listening on {addr:?}"); + let svc = FlightServiceServer::new(service); + + Server::builder().add_service(svc).serve(addr).await?; + + Ok(()) +} + +pub struct FlightSqlServiceImpl { + contexts: Arc>>, + statements: Arc>, + results: Arc>>, + ctx: Arc>, +} + +impl FlightSqlServiceImpl { + async fn create_ctx(&self) -> Result { + let uuid = Uuid::new_v4().hyphenated().to_string(); + let session_config = SessionConfig::from_env() + .map_err(|e| Status::internal(format!("Error building plan: {e}")))? + .with_information_schema(true); + let ctx = Arc::new(SessionContext::new_with_config(session_config)); + + self.contexts.insert(uuid.clone(), ctx); + Ok(uuid) + } + + fn get_ctx(&self, req: &Request) -> Result, Status> { + // get the token from the authorization header on Request + let auth = req + .metadata() + .get("authorization") + .ok_or_else(|| Status::internal("No authorization header!"))?; + let str = auth + .to_str() + .map_err(|e| Status::internal(format!("Error parsing header: {e}")))?; + let authorization = str.to_string(); + let bearer = "Bearer "; + if !authorization.starts_with(bearer) { + Err(Status::internal("Invalid auth header!"))?; + } + let auth = authorization[bearer.len()..].to_string(); + + if let Some(context) = self.contexts.get(&auth) { + Ok(context.clone()) + } else { + Err(Status::internal(format!( + "Context handle not found: {auth}" + )))? + } + } + + fn get_plan(&self, handle: &str) -> Result { + if let Some(plan) = self.statements.get(handle) { + Ok(plan.clone()) + } else { + Err(Status::internal(format!("Plan handle not found: {handle}")))? + } + } + + fn get_result(&self, handle: &str) -> Result, Status> { + if let Some(result) = self.results.get(handle) { + Ok(result.clone()) + } else { + Err(Status::internal(format!( + "Request handle not found: {handle}" + )))? + } + } + + async fn tables(&self, ctx: Arc) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("catalog_name", DataType::Utf8, true), + Field::new("db_schema_name", DataType::Utf8, true), + Field::new("table_name", DataType::Utf8, false), + Field::new("table_type", DataType::Utf8, false), + ])); + + let mut catalogs = vec![]; + let mut schemas = vec![]; + let mut names = vec![]; + let mut types = vec![]; + for catalog in ctx.catalog_names() { + let catalog_provider = ctx.catalog(&catalog).unwrap(); + for schema in catalog_provider.schema_names() { + let schema_provider = catalog_provider.schema(&schema).unwrap(); + for table in schema_provider.table_names() { + let table_provider = schema_provider.table(&table).await.unwrap().unwrap(); + catalogs.push(catalog.clone()); + schemas.push(schema.clone()); + names.push(table.clone()); + types.push(table_provider.table_type().to_string()) + } + } + } + + RecordBatch::try_new( + schema, + [catalogs, schemas, names, types] + .into_iter() + .map(|i| Arc::new(StringArray::from(i)) as ArrayRef) + .collect::>(), + ) + .unwrap() + } + + fn remove_plan(&self, handle: &str) -> Result<(), Status> { + self.statements.remove(&handle.to_string()); + Ok(()) + } + + fn remove_result(&self, handle: &str) -> Result<(), Status> { + self.results.remove(&handle.to_string()); + Ok(()) + } +} + +#[tonic::async_trait] +impl FlightSqlService for FlightSqlServiceImpl { + type FlightService = FlightSqlServiceImpl; + + // This function will be triggered if client JDBC property's `user` and `password` field set + async fn do_handshake( + &self, + _request: Request>, + ) -> Result< + Response> + Send>>>, + Status, + > { + info!("do_handshake"); + if let Some(msg) = _request.metadata().get("create") { + // A new round start at SQLancer, clear the ctx + info!("Resetting ctx {:?}", msg); + let new_ctx = + SessionContext::new_with_config(SessionConfig::new().with_information_schema(true)); + + let mut ctx_guard = self.ctx.lock().await; // Use `lock()` for async Mutex + *ctx_guard = new_ctx; + + // Clear leaked state from previous round + self.statements.clear(); + self.results.clear(); + self.contexts.clear(); + } + // no authentication actually takes place here + // see Ballista implementation for example of basic auth + // in this case, we simply accept the connection and create a new SessionContext + // the SessionContext will be re-used within this same connection/session + let token = self.create_ctx().await?; + + let result = HandshakeResponse { + protocol_version: 0, + payload: token.as_bytes().to_vec().into(), + }; + let result = Ok(result); + let output = futures::stream::iter(vec![result]); + let str = format!("Bearer {token}"); + let mut resp: Response> + Send>>> = + Response::new(Box::pin(output)); + let md = MetadataValue::try_from(str) + .map_err(|_| Status::invalid_argument("authorization not parsable"))?; + resp.metadata_mut().insert("authorization", md); + Ok(resp) + } + + async fn do_get_fallback( + &self, + _request: Request, + message: Any, + ) -> Result::DoGetStream>, Status> { + if !message.is::() { + Err(Status::unimplemented(format!( + "do_get: The defined request is invalid: {}", + message.type_url + )))? + } + + let fr: FetchResults = message + .unpack() + .map_err(|e| Status::internal(format!("{e:?}")))? + .ok_or_else(|| Status::internal("Expected FetchResults but got None!"))?; + + let handle = fr.handle; + + info!("getting results for {handle}"); + let result = self.get_result(&handle)?; + // if we get an empty result, create an empty schema + let (schema, batches) = match result.first() { + None => (Arc::new(Schema::empty()), vec![]), + Some(batch) => (batch.schema(), result.clone()), + }; + + let batch_stream = futures::stream::iter(batches).map(Ok); + + let stream = FlightDataEncoderBuilder::new() + .with_schema(schema) + .build(batch_stream) + .map_err(Status::from); + + Ok(Response::new(Box::pin(stream))) + } + + async fn get_flight_info_prepared_statement( + &self, + cmd: CommandPreparedStatementQuery, + _request: Request, + ) -> Result, Status> { + info!("get_flight_info_prepared_statement {:?}", cmd); + let handle = std::str::from_utf8(&cmd.prepared_statement_handle) + .map_err(|e| status!("Unable to parse uuid", e))?; + + //let ctx = self.get_ctx(&request)?; + let plan = self.get_plan(handle)?; + + let ctx_guard = self.ctx.lock().await; + let state = (*ctx_guard).state(); + let df = DataFrame::new(state, plan); + let result = df + .collect() + .await + .map_err(|e| status!("Error executing query", e))?; + + // if we get an empty result, create an empty schema + let schema = match result.first() { + None => Schema::empty(), + Some(batch) => (*batch.schema()).clone(), + }; + + self.results.insert(handle.to_string(), result); + + // if we had multiple endpoints to connect to, we could use this Location + // but in the case of standalone DataFusion, we don't + // let loc = Location { + // uri: "grpc+tcp://127.0.0.1:50051".to_string(), + // }; + let fetch = FetchResults { + handle: handle.to_string(), + }; + let buf = fetch.as_any().encode_to_vec().into(); + let ticket = Ticket { ticket: buf }; + + let info = FlightInfo::new() + // Encode the Arrow schema + .try_with_schema(&schema) + .expect("encoding failed") + .with_endpoint(FlightEndpoint::new().with_ticket(ticket)) + .with_descriptor(FlightDescriptor { + r#type: DescriptorType::Cmd.into(), + cmd: Default::default(), + path: vec![], + }); + let resp = Response::new(info); + Ok(resp) + } + + async fn get_flight_info_tables( + &self, + _query: CommandGetTables, + request: Request, + ) -> Result, Status> { + info!("get_flight_info_tables"); + let ctx = self.get_ctx(&request)?; + let data = self.tables(ctx).await; + let schema = data.schema(); + + let uuid = Uuid::new_v4().hyphenated().to_string(); + self.results.insert(uuid.clone(), vec![data]); + + let fetch = FetchResults { handle: uuid }; + let buf = fetch.as_any().encode_to_vec().into(); + let ticket = Ticket { ticket: buf }; + + let info = FlightInfo::new() + // Encode the Arrow schema + .try_with_schema(&schema) + .expect("encoding failed") + .with_endpoint(FlightEndpoint::new().with_ticket(ticket)) + .with_descriptor(FlightDescriptor { + r#type: DescriptorType::Cmd.into(), + cmd: Default::default(), + path: vec![], + }); + let resp = Response::new(info); + Ok(resp) + } + + async fn do_put_prepared_statement_update( + &self, + handle: CommandPreparedStatementUpdate, + _request: Request, + ) -> Result { + info!("do_put_prepared_statement_update"); + // statements like "CREATE TABLE.." or "SET datafusion.nnn.." call this function + // and we are required to return some row count here + let handle = std::str::from_utf8(&handle.prepared_statement_handle) + .map_err(|e| status!("Unable to parse uuid", e))?; + + //let ctx = self.get_ctx(&request)?; + let plan = self.get_plan(handle)?; + //println!("do_put_prepared_statement_update plan is {:?}", plan); + + let ctx_guard = self.ctx.lock().await; + let state = (*ctx_guard).state(); + let df = DataFrame::new(state, plan); + df.collect() + .await + .map_err(|e| status!("Error executing query", e))?; + + Ok(1) + } + + async fn do_action_create_prepared_statement( + &self, + query: ActionCreatePreparedStatementRequest, + _request: Request, + ) -> Result { + let user_query = query.query.as_str(); + info!("do_action_create_prepared_statement: {user_query}"); + + //let ctx = self.get_ctx(&request)?; + + let ctx_guard = self.ctx.lock().await; + let plan = (*ctx_guard) + .sql(user_query) + .await + .and_then(|df| df.into_optimized_plan()) + .map_err(|e| Status::internal(format!("Error building plan: {e}")))?; + + // store a copy of the plan, it will be used for execution + let plan_uuid = Uuid::new_v4().hyphenated().to_string(); + self.statements.insert(plan_uuid.clone(), plan.clone()); + + let plan_schema = plan.schema(); + + let arrow_schema = (&**plan_schema).into(); + let message = SchemaAsIpc::new(&arrow_schema, &IpcWriteOptions::default()) + .try_into() + .map_err(|e| status!("Unable to serialize schema", e))?; + let IpcMessage(schema_bytes) = message; + + let res = ActionCreatePreparedStatementResult { + prepared_statement_handle: plan_uuid.into(), + dataset_schema: schema_bytes, + parameter_schema: Default::default(), + }; + Ok(res) + } + + async fn do_action_close_prepared_statement( + &self, + handle: ActionClosePreparedStatementRequest, + _request: Request, + ) -> Result<(), Status> { + info!("do_action_close_prepared_statement"); + let handle = std::str::from_utf8(&handle.prepared_statement_handle); + if let Ok(handle) = handle { + info!("do_action_close_prepared_statement: removing plan and results for {handle}"); + let _ = self.remove_plan(handle); + let _ = self.remove_result(handle); + } + Ok(()) + } + + async fn register_sql_info(&self, _id: i32, _result: &SqlInfo) {} +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FetchResults { + #[prost(string, tag = "1")] + pub handle: ::prost::alloc::string::String, +} + +impl ProstMessageExt for FetchResults { + fn type_url() -> &'static str { + "type.googleapis.com/datafusion.example.com.sql.FetchResults" + } + + fn as_any(&self) -> Any { + Any { + type_url: FetchResults::type_url().to_string(), + value: ::prost::Message::encode_to_vec(self).into(), + } + } +} diff --git a/src/sqlancer/doris/DorisBugs.java b/src/sqlancer/doris/DorisBugs.java new file mode 100644 index 000000000..956be1683 --- /dev/null +++ b/src/sqlancer/doris/DorisBugs.java @@ -0,0 +1,45 @@ +package sqlancer.doris; + +public final class DorisBugs { + // https://github.com/apache/doris/issues/19370 + // Internal Error occur in GroupBy&Having sql + // fixed by https://github.com/apache/doris/pull/19559 + public static boolean bug19370; + + // https://github.com/apache/doris/issues/19374 + // Different result of having not ($value in column) and having ($value not in column) + // fixed by https://github.com/apache/doris/pull/19471 + public static boolean bug19374; + + // https://github.com/apache/doris/issues/19611 + // ERROR occur in nested subqueries with same column name and union + public static boolean bug19611 = true; + + // https://github.com/apache/doris/issues/36070 + // Expression evaluate to NULL but is treated as FALSE in where clause + public static boolean bug36070 = true; + + // https://github.com/apache/doris/issues/36072 + // SELECT DISTINCT does not work with aggregate key column + public static boolean bug36072 = true; + + // https://github.com/apache/doris/issues/36342 + // Wrong result with INNER JOIN and CURRENT_TIMESTAMP + public static boolean bug36342 = true; + + // https://github.com/apache/doris/issues/36343 + // Wrong result with SELECT DISTINCT and UNIQUE model + public static boolean bug36343 = true; + + // https://github.com/apache/doris/issues/36346 + // Wrong result with LEFT JOIN SELECT DISTINCT and IN operation + public static boolean bug36346 = true; + + // https://github.com/apache/doris/issues/36351 + // Wrong result with TINYINT column with value -1049190528 + public static boolean bug36351 = true; + + private DorisBugs() { + + } +} diff --git a/src/sqlancer/doris/DorisErrors.java b/src/sqlancer/doris/DorisErrors.java new file mode 100644 index 000000000..28e93b80a --- /dev/null +++ b/src/sqlancer/doris/DorisErrors.java @@ -0,0 +1,85 @@ +package sqlancer.doris; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class DorisErrors { + + private DorisErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + // SQL syntax error + errors.add("Syntax error"); + errors.add("Please check your sql, we meet an error when parsing"); + errors.add("but returns type"); + errors.add("is not a number"); + + // Not in line with Doris' logic + errors.add("Unexpected exception: null"); + errors.add("Cross join can't be used with ON clause"); + errors.add("BetweenPredicate needs to be rewritten into a CompoundPredicate"); + errors.add("can't be assigned to some PlanNode"); + errors.add("can not cast from origin type"); + errors.add("not produced by aggregation output"); + errors.add("cannot combine"); // cannot combine SELECT DISTINCT with aggregate functions or GROUP BY + errors.add("Invalid type"); + errors.add("cannot be cast to"); + + // functions + errors.add("No matching function with signature"); + errors.add("Invalid number format"); + errors.add("group_concat requires"); + errors.add("function's argument should be"); + errors.add("requires a numeric parameter"); + errors.add("out of bounds"); + errors.add("function do not support"); + errors.add("parameter must be"); + errors.add("Not supported input arguments types"); + errors.add("No matching function with signature"); + errors.add("function"); + errors.add("Invalid"); + errors.add("Incorrect"); + + // regex + + // To avoid bugs + if (DorisBugs.bug19370) { + errors.add("failed to initialize storage"); + } + if (DorisBugs.bug19374) { + errors.add("the size of the result sets mismatch"); + } + if (DorisBugs.bug19611) { + errors.add("Duplicated inline view column alias"); + } + errors.add("Arithmetic overflow"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Insert has filtered data in strict mode"); + errors.add("Only value columns of unique table could be updated"); + errors.add("Only unique olap table could be updated"); + errors.add("Number out of range"); + errors.add("Arithmetic overflow"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + +} diff --git a/src/sqlancer/doris/DorisOptions.java b/src/sqlancer/doris/DorisOptions.java new file mode 100644 index 000000000..7e5a2c7e3 --- /dev/null +++ b/src/sqlancer/doris/DorisOptions.java @@ -0,0 +1,106 @@ +package sqlancer.doris; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(commandDescription = "Apache Doris (default port: " + DorisOptions.DEFAULT_PORT + ", default host: " + + DorisOptions.DEFAULT_HOST + ")") +public class DorisOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 9030; + + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/views that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions", arity = 1) + public boolean testBetween = true; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-decimal-constants", description = "Allow generating DECIMAL constants", arity = 1) + public boolean testDecimalConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-datetime-constants", description = "Allow generating DATETIME constants", arity = 1) + public boolean testDateTimeConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates; + + @Parameter(names = "--max-num-table-alters", description = "The maximum number of ALTER TABLE statements that are issued for a database", arity = 1) + public int maxNumTableAlters; + + @Parameter(names = "--test-engine-type", description = "The engine type in Doris, only consider OLAP now", arity = 1) + public String testEngineType = "OLAP"; // skip now + + @Parameter(names = "--test-indexes", description = "Allow explicit indexes, Doris only supports creating indexes on single-column BITMAP", arity = 1) + public boolean testIndexes = true; // skip now + + @Parameter(names = "--test-column-aggr", description = "Allow test column aggregation (sum, min, max, replace, replace_if_not_null, hll_union, bitmap_untion)", arity = 1) + public boolean testColumnAggr = true; + + @Parameter(names = "--test-datemodel", description = "Allow generating Doris’s data model in tables. (Aggregate、Uniqe、Duplicate)", arity = 1) + public boolean testDataModel = true; + + @Parameter(names = "--test-distribution", description = "Allow generating data distribution in tables.", arity = 1) + public boolean testDistribution = true; // must have it, skip now + + @Parameter(names = "--test-rollup", description = "Allow generating rollups in tables.", arity = 1) + public boolean testRollup = true; // skip now + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(DorisOracleFactory.NOREC); + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/doris/DorisOracleFactory.java b/src/sqlancer/doris/DorisOracleFactory.java new file mode 100644 index 000000000..8f28f0f21 --- /dev/null +++ b/src/sqlancer/doris/DorisOracleFactory.java @@ -0,0 +1,108 @@ +package sqlancer.doris; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.oracle.DorisPivotedQuerySynthesisOracle; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningAggregateTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningDistinctTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningGroupByTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningHavingTester; + +public enum DorisOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with(DorisErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + PQS { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + return new DorisPivotedQuerySynthesisOracle(globalState); + } + }, + ALL { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(NOREC.create(globalState)); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(new DorisPivotedQuerySynthesisOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/doris/DorisProvider.java b/src/sqlancer/doris/DorisProvider.java new file mode 100644 index 000000000..3f64231f1 --- /dev/null +++ b/src/sqlancer/doris/DorisProvider.java @@ -0,0 +1,155 @@ +package sqlancer.doris; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.gen.DorisAlterTableGenerator; +import sqlancer.doris.gen.DorisDeleteGenerator; +import sqlancer.doris.gen.DorisDropTableGenerator; +import sqlancer.doris.gen.DorisDropViewGenerator; +import sqlancer.doris.gen.DorisIndexGenerator; +import sqlancer.doris.gen.DorisInsertGenerator; +import sqlancer.doris.gen.DorisTableGenerator; +import sqlancer.doris.gen.DorisUpdateGenerator; +import sqlancer.doris.gen.DorisViewGenerator; + +@AutoService(DatabaseProvider.class) +public class DorisProvider extends SQLProviderAdapter { + + public DorisProvider() { + super(DorisGlobalState.class, DorisOptions.class); + } + + public enum Action implements AbstractAction { + CREATE_TABLE(DorisTableGenerator::createRandomTableStatement), CREATE_VIEW(DorisViewGenerator::getQuery), + CREATE_INDEX(DorisIndexGenerator::getQuery), INSERT(DorisInsertGenerator::getQuery), + DELETE(DorisDeleteGenerator::generate), UPDATE(DorisUpdateGenerator::getQuery), + ALTER_TABLE(DorisAlterTableGenerator::getQuery), + TRUNCATE((g) -> new SQLQueryAdapter( + "TRUNCATE TABLE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), + DROP_TABLE(DorisDropTableGenerator::dropTable), DROP_VIEW(DorisDropViewGenerator::dropView); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(DorisGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(DorisGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes); + case UPDATE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates); + case ALTER_TABLE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumTableAlters); + case TRUNCATE: + return r.getInteger(0, 2); + case CREATE_TABLE: + case CREATE_INDEX: + case CREATE_VIEW: + case DROP_TABLE: + case DROP_VIEW: + return 0; + default: + throw new AssertionError(a); + } + } + + public static class DorisGlobalState extends SQLGlobalState { + + @Override + protected DorisSchema readSchema() throws SQLException { + return DorisSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(DorisGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success = false; + do { + SQLQueryAdapter qt = new DorisTableGenerator().getQuery(globalState); + if (qt != null) { + success = globalState.executeStatement(qt); + } + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + DorisProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(DorisGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + if (password.equals("\"\"")) { + password = ""; + } + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = DorisOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = DorisOptions.DEFAULT_PORT; + } + String databaseName = globalState.getDatabaseName(); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "doris"; + } + +} diff --git a/src/sqlancer/doris/DorisSchema.java b/src/sqlancer/doris/DorisSchema.java new file mode 100644 index 000000000..70a61ee62 --- /dev/null +++ b/src/sqlancer/doris/DorisSchema.java @@ -0,0 +1,619 @@ +package sqlancer.doris; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.ast.DorisConstant; + +public class DorisSchema extends AbstractSchema { + + public enum DorisTableDataModel { + UNIQUE, AGGREGATE, DUPLICATE; + + public static DorisTableDataModel getRandom() { + List validOptions = new ArrayList<>(Arrays.asList(values())); + if (DorisBugs.bug36072) { + validOptions.remove(AGGREGATE); + } + if (DorisBugs.bug36343) { + validOptions.remove(UNIQUE); + } + return Randomly.fromList(validOptions); + } + } + + public enum DorisColumnAggrType { + SUM, MIN, MAX, REPLACE, REPLCAE_IF_NOT_NULL, BITMAP_UNION, HLL_UNION, NULL; + + public static DorisColumnAggrType getRandom(DorisCompositeDataType columnDataType) { + // if (columnDataType.getPrimitiveDataType() == DorisSchema.DorisDataType.BITMAP) { + // return DorisColumnAggrType.BITMAP_UNION; + // } + // if (columnDataType.getPrimitiveDataType() == DorisSchema.DorisDataType.HLL) { + // return DorisColumnAggrType.HLL_UNION; + // } + + return Randomly.fromOptions(SUM, MIN, MAX, REPLACE, REPLCAE_IF_NOT_NULL); + } + } + + public enum DorisDataType { + INT, FLOAT, DECIMAL, DATE, DATETIME, VARCHAR, BOOLEAN, NULL; + // HLL, BITMAP, ARRAY; + + private int decimalScale; + private int decimalPrecision; + private int varcharLength; + + public static DorisDataType getRandomWithoutNull() { + DorisDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DorisDataType.NULL); + return dt; + } + + public int getDecimalScale() { + return decimalScale; + } + + public void setDecimalScale(int decimalScale) { + this.decimalScale = decimalScale; + } + + public int getDecimalPrecision() { + return decimalPrecision; + } + + public void setDecimalPrecision(int decimalPrecision) { + this.decimalPrecision = decimalPrecision; + } + + public int getVarcharLength() { + return varcharLength; + } + + public void setVarcharLength(int varcharLength) { + this.varcharLength = varcharLength; + } + } + + public static class DorisCompositeDataType { + + private final DorisDataType dataType; + + private final int size; + + public DorisCompositeDataType(DorisDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public DorisDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static DorisCompositeDataType getRandomWithoutNull() { + DorisDataType type = DorisDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8, 16); + break; + case FLOAT: + size = Randomly.fromOptions(4, 12); + break; + case DECIMAL: + size = Randomly.fromOptions(1, 3); // DECIMAL or DECIMALV3 + break; + case DATE: + case DATETIME: + case VARCHAR: + case BOOLEAN: + // case HLL: + // case BITMAP: + // case ARRAY: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new DorisCompositeDataType(type, size); + } + + public void initColumnArgs() { + Randomly r = new Randomly(); + int scale; + int precision; + int varcharLength; + switch (getPrimitiveDataType()) { + case DECIMAL: + if (getPrimitiveDataType().getDecimalPrecision() != 0) { + break; + } + if (size == 1) { + scale = r.getInteger(0, 9); + precision = r.getInteger(scale + 1, scale + 18); + getPrimitiveDataType().setDecimalPrecision(precision); + getPrimitiveDataType().setDecimalScale(scale); + } else if (size == 3) { + precision = r.getInteger(1, 38); + scale = r.getInteger(0, precision); + getPrimitiveDataType().setDecimalPrecision(precision); + getPrimitiveDataType().setDecimalScale(scale); + } else { + throw new AssertionError(size); + } + break; + case VARCHAR: + if (getPrimitiveDataType().getVarcharLength() != 0) { + break; + } + varcharLength = r.getInteger(1, 255); + getPrimitiveDataType().setVarcharLength(varcharLength); + break; + default: + // pass + } + + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 16: + return "LARGEINT"; + case 8: + return "BIGINT"; + case 4: + return "INT"; + case 2: + return "SMALLINT"; + case 1: + return "TINYINT"; + default: + throw new AssertionError(size); + } + case FLOAT: + switch (size) { + case 12: + return "DOUBLE"; + case 4: + return "FLOAT"; + default: + throw new AssertionError(size); + } + case DECIMAL: + switch (size) { + case 1: + return "DECIMAL(" + getPrimitiveDataType().getDecimalPrecision() + "," + + getPrimitiveDataType().getDecimalScale() + ")"; + case 3: + return "DECIMALV3(" + getPrimitiveDataType().getDecimalPrecision() + "," + + getPrimitiveDataType().getDecimalScale() + ")"; + default: + throw new AssertionError(size); + } + case DATE: + return "DATEV2"; + case DATETIME: + return Randomly.fromOptions("DATETIME", "DATETIMEV2"); + case VARCHAR: + return Randomly.fromOptions("VARCHAR", "CHAR") + "(" + getPrimitiveDataType().getVarcharLength() + ")"; + case BOOLEAN: + return "BOOLEAN"; + // case HLL: + // return "HLL"; + // case BITMAP: + // return "BITMAP"; + // case ARRAY: + // return "ARRAY"; + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + public boolean canBeKey() { + switch (dataType) { + // case HLL: + // case BITMAP: + // case ARRAY: + case FLOAT: + return false; + default: + return true; + } + } + + } + + public static class DorisColumn extends AbstractTableColumn { + + private final boolean isKey; + private final boolean isNullable; + private final DorisColumnAggrType aggrType; + private final boolean hasDefaultValue; + private final String defaultValue; + + public DorisColumn(String name, DorisCompositeDataType type, boolean isKey, boolean isNullable, + DorisColumnAggrType aggrType, boolean hasDefaultValue, String defaultValue) { + super(name, null, type); + this.isKey = isKey; + this.isNullable = isNullable; + this.aggrType = aggrType; + this.hasDefaultValue = hasDefaultValue; + this.defaultValue = defaultValue; + } + + public DorisColumn(String name, DorisCompositeDataType type, boolean isKey, boolean isNullable) { + super(name, null, type); + this.isKey = isKey; + this.isNullable = isNullable; + this.aggrType = DorisColumnAggrType.NULL; + this.hasDefaultValue = false; + this.defaultValue = ""; + } + + public boolean isKey() { + return isKey; + } + + public boolean isNullable() { + return isNullable; + } + + public boolean hasDefaultValue() { + return hasDefaultValue; + } + + @Override + public String toString() { + String ret = this.getName() + " " + this.getType(); + if (aggrType != DorisColumnAggrType.NULL) { + ret += " " + aggrType.name(); + } + if (!isNullable) { + ret += " NOT NULL"; + } + if (hasDefaultValue) { + ret += " DEFAULT " + defaultValue; + } + return ret; + } + + @Override + public int compareTo(AbstractTableColumn o) { + // To sort columns + DorisColumn other = (DorisColumn) o; + if (isKey != other.isKey) { + return isKey ? 1 : -1; + } + return getName().compareTo(other.getName()); + } + } + + public static class DorisTables extends AbstractTables { + + public DorisTables(List tables) { + super(tables); + } + + public DorisRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String rowValueQuery = String.format("SELECT %s FROM %s ORDER BY 1 LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery(rowValueQuery); + if (!rs.next()) { + throw new IgnoreMeException(); + // throw new AssertionError("could not find random row " + rowValueQuery + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + DorisColumn column = getColumns().get(i); + int columnIndex = rs.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + DorisConstant constant; + if (rs.getString(columnIndex) == null) { + constant = DorisConstant.createNullConstant(); + } else { + switch (column.getType().getPrimitiveDataType()) { + case INT: + constant = DorisConstant.createIntConstant(rs.getLong(columnIndex)); + break; + case FLOAT: + case DECIMAL: + constant = DorisConstant.createFloatConstant(rs.getDouble(columnIndex)); + break; + case DATE: + constant = DorisConstant.createDateConstant(rs.getString(columnIndex)); + break; + case DATETIME: + constant = DorisConstant.createDatetimeConstant(rs.getString(columnIndex)); + break; + case VARCHAR: + constant = DorisConstant.createStringConstant(rs.getString(columnIndex)); + break; + case BOOLEAN: + constant = DorisConstant.createBooleanConstant(rs.getBoolean(columnIndex)); + break; + case NULL: + constant = DorisConstant.createNullConstant(); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !rs.next(); + return new DorisRowValue(this, values); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + + } + + public static class DorisRowValue extends AbstractRowValue { + + DorisRowValue(DorisTables tables, Map values) { + super(tables, values); + } + + } + + public DorisSchema(List databaseTables) { + super(databaseTables); + } + + public DorisTables getRandomTableNonEmptyTables() { + return new DorisTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public DorisTables getRandomTableNonEmptyAndViewTables() { + List tables = getDatabaseTables().stream().filter(t -> !t.isView()).collect(Collectors.toList()); + tables = Randomly.nonEmptySubset(tables); + return new DorisTables(tables); + } + + public int getIndexCount() { + int count = 0; + for (DorisTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + + private static DorisCompositeDataType getColumnType(String typeString) { + DorisDataType primitiveType; + int size = -1; + + if (typeString.startsWith("DECIMALV3")) { + primitiveType = DorisDataType.DECIMAL; + String precisionAndScale = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + String[] split = precisionAndScale.split(","); + assert split.length == 2; + primitiveType.setDecimalPrecision(Integer.parseInt(split[0].trim())); + primitiveType.setDecimalScale(Integer.parseInt(split[1].trim())); + size = 3; + } else if (typeString.startsWith("DECIMAL")) { + primitiveType = DorisDataType.DECIMAL; + String precisionAndScale = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + String[] split = precisionAndScale.split(","); + assert split.length == 2; + primitiveType.setDecimalPrecision(Integer.parseInt(split[0].trim())); + primitiveType.setDecimalScale(Integer.parseInt(split[1].trim())); + size = 1; + } else if (typeString.startsWith("DATEV2")) { + primitiveType = DorisDataType.DATE; + size = 2; + } else if (typeString.startsWith("DATE")) { + primitiveType = DorisDataType.DATE; + size = 1; + } else if (typeString.startsWith("DATETIMEV2")) { + primitiveType = DorisDataType.DATETIME; + size = 2; + } else if (typeString.startsWith("DATETIME")) { + primitiveType = DorisDataType.DATETIME; + size = 1; + } else if (typeString.startsWith("CHAR") || typeString.startsWith("VARCHAR")) { + primitiveType = DorisDataType.VARCHAR; + String varcharLength = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + primitiveType.setVarcharLength(Integer.parseInt(varcharLength.trim())); + } else { + switch (typeString) { + case "LARGEINT": + primitiveType = DorisDataType.INT; + size = 16; + break; + case "BIGINT": + primitiveType = DorisDataType.INT; + size = 8; + break; + case "INT": + primitiveType = DorisDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = DorisDataType.INT; + size = 2; + break; + case "TINYINT": + primitiveType = DorisDataType.INT; + size = 1; + break; + case "DOUBLE": + primitiveType = DorisDataType.FLOAT; + size = 12; + break; + case "FLOAT": + primitiveType = DorisDataType.FLOAT; + size = 4; + break; + case "DECIMAL": + case "DECIMAL(*,*)": + primitiveType = DorisDataType.DECIMAL; + size = 1; + break; + case "DECIMALV3": + case "DECIMALV3(*,*)": + primitiveType = DorisDataType.DECIMAL; + size = 3; + break; + case "CHAR": + case "CHAR(*)": + case "VARCHAR": + case "VARCHAR(*)": + primitiveType = DorisDataType.VARCHAR; + break; + case "DATE": + primitiveType = DorisDataType.DATE; + size = 1; + break; + case "DATEV2": + primitiveType = DorisDataType.DATE; + size = 2; + break; + case "DATETIME": + primitiveType = DorisDataType.DATETIME; + size = 1; + break; + case "DATETIMEV2": + primitiveType = DorisDataType.DATETIME; + size = 2; + break; + case "BOOLEAN": + primitiveType = DorisDataType.BOOLEAN; + break; + // case "HLL": + // primitiveType = DorisDataType.HLL; + // break; + // case "BITMAP": + // primitiveType = DorisDataType.BITMAP; + // break; + case "NULL": + primitiveType = DorisDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + } + return new DorisCompositeDataType(primitiveType, size); + } + + public static class DorisTable extends AbstractRelationalTable { + + public DorisTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + public List getRandomNonEmptyInsertColumns() { + List columns = getColumns(); + List retColumns = new ArrayList<>(); + List remainColumns = new ArrayList<>(); + for (DorisColumn column : columns) { + if (!column.hasDefaultValue() && !column.isNullable) { + retColumns.add(column); + } else { + remainColumns.add(column); + } + } + if (retColumns.isEmpty()) { + retColumns.addAll(Randomly.nonEmptySubset(remainColumns)); + } else { + retColumns.addAll(Randomly.subset(remainColumns)); + } + return retColumns; + } + + } + + public static DorisSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = matchesViewName(tableName); + DorisTable t = new DorisTable(tableName, databaseColumns, isView); + for (DorisColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new DorisSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES")) { + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("DESCRIBE " + tableName)) { + while (rs.next()) { + String columnName = rs.getString("Field"); + String dataType = rs.getString("Type"); + String isNullString = rs.getString("Null"); + assert isNullString.contentEquals("Yes") || isNullString.contentEquals("No"); + boolean isNullable = isNullString.contentEquals("Yes"); + String isKeyString = rs.getString("Key"); + assert isKeyString.contentEquals("true") || isKeyString.contentEquals("false"); + boolean isKey = isKeyString.contentEquals("true"); + String defaultValue = rs.getString("Default"); + boolean hasDefaultValue = defaultValue != null; + DorisColumn c = new DorisColumn(columnName, getColumnType(dataType), isKey, isNullable, + DorisColumnAggrType.NULL, hasDefaultValue, defaultValue); + columns.add(c); + } + } + } + return columns; + } + +} diff --git a/src/sqlancer/doris/ast/DorisAggregateOperation.java b/src/sqlancer/doris/ast/DorisAggregateOperation.java new file mode 100644 index 000000000..4c60985a8 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisAggregateOperation.java @@ -0,0 +1,37 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.Randomly; + +public class DorisAggregateOperation extends DorisFunction + implements DorisExpression { + + public DorisAggregateOperation(List args, DorisAggregateFunction func) { + super(args, func); + } + + public enum DorisAggregateFunction { + COLLECT_SET(1), MIN(1), STDDEV_SAMP(1), AVG(1), AVG_WEIGHTED(2), PERCENTILE(1), PERCENTILE_ARRAY(2), + HLL_UNION_AGG(1), TOPN(2), TOPN_ARRAY(2), TOPN_WEIGHTED(3), COUNT(1), SUM(1), MAX_BY(2), BITMAP_UNION(1), + GROUP_BITMAP_XOR(1), GROUP_BIT_ADD(1), GROUP_BIT_OR(1), GROUP_BIT_XOR(1), PERCENTILE_APPROX(2), STDDEV(1), + STDDEV_POP(1), GROUP_CONCAT(1), COLLECT_LIST(1), MIN_BY(2), MAX(1), ANY_VALUE(1), VAR_SAMP(1), VARIANCE_SAMP(1), + APPROX_COUNT_DISTINCT(1), VARIANCE(1), VAR_POP(1), VARIANCE_POP(1), GROUPING(1), GROUPING_ID(1); + // RETENTION(1), SEQUENCE_MATCH(1), SEQUENCE_COUNT(1), // TODO，not currently considered + + private int nrArgs; + + DorisAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static DorisAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } +} diff --git a/src/sqlancer/doris/ast/DorisAlias.java b/src/sqlancer/doris/ast/DorisAlias.java new file mode 100644 index 000000000..74acf1cff --- /dev/null +++ b/src/sqlancer/doris/ast/DorisAlias.java @@ -0,0 +1,9 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DorisAlias extends NewAliasNode implements DorisExpression { + public DorisAlias(DorisExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/doris/ast/DorisBetweenOperation.java b/src/sqlancer/doris/ast/DorisBetweenOperation.java new file mode 100644 index 000000000..96c0b428c --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBetweenOperation.java @@ -0,0 +1,38 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.doris.DorisSchema; + +public class DorisBetweenOperation extends NewBetweenOperatorNode implements DorisExpression { + public DorisBetweenOperation(DorisExpression left, DorisExpression middle, DorisExpression right, boolean isTrue) { + super(left, middle, right, isTrue); + } + + public DorisExpression getLeftExpr() { + return left; + } + + public DorisExpression getMiddleExpr() { + return middle; + } + + public DorisExpression getRightExpr() { + return right; + } + + @Override + public DorisConstant getExpectedValue() { + DorisBinaryComparisonOperation leftComparison = new DorisBinaryComparisonOperation(getMiddleExpr(), + getLeftExpr(), DorisBinaryComparisonOperation.DorisBinaryComparisonOperator.LESS_EQUALS); + DorisBinaryComparisonOperation rightComparison = new DorisBinaryComparisonOperation(getLeftExpr(), + getRightExpr(), DorisBinaryComparisonOperation.DorisBinaryComparisonOperator.LESS_EQUALS); + return new DorisBinaryLogicalOperation(leftComparison, rightComparison, + DorisBinaryLogicalOperation.DorisBinaryLogicalOperator.AND).getExpectedValue(); + } + + @Override + public DorisSchema.DorisDataType getExpectedType() { + return DorisSchema.DorisDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java new file mode 100644 index 000000000..2e148c488 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java @@ -0,0 +1,142 @@ +package sqlancer.doris.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisBinaryArithmeticOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryArithmeticOperation(DorisExpression left, DorisExpression right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } + + public enum DorisBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADDITION("+") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l + r); + } + }, + SUBTRACTION("-") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + } + }, + MODULO("%") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + } + }, + CONCAT("||") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isBoolean() || !right.isBoolean()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> l == 1 || r == 1 ? 1.0 : 0.0); + } + }, + BIT_AND("&") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() & (int) r.doubleValue())); + } + }, + BIT_OR("|") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() | (int) r.doubleValue())); + } + }, + LSHIFT("<<") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() << (int) r.doubleValue())); + } + }, + RSHIFT(">>") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() >> (int) r.doubleValue())); + } + }; + + private final String textRepresentation; + + DorisBinaryArithmeticOperator(String text) { + textRepresentation = text; + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + public DorisConstant applyOperation(DorisConstant left, DorisConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + double leftVal = left.cast(DorisDataType.FLOAT).asFloat(); + double rightVal = right.cast(DorisDataType.FLOAT).asFloat(); + return DorisConstant.createFloatConstant(op.apply(leftVal, rightVal)); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + public DorisExpression getLeftExpr() { + return super.getLeft(); + } + + public DorisExpression getRightExpr() { + return super.getRight(); + } + + public DorisBinaryArithmeticOperator getOp() { + return (DorisBinaryArithmeticOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = getLeftExpr().getExpectedValue(); + DorisConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.FLOAT; + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java new file mode 100644 index 000000000..13355c7ff --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java @@ -0,0 +1,119 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisBinaryComparisonOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryComparisonOperation(DorisExpression left, DorisExpression right, + DorisBinaryComparisonOperator op) { + super(left, right, op); + } + + public DorisExpression getLeftExpression() { + return super.getLeft(); + } + + public DorisExpression getRightExpression() { + return super.getRight(); + } + + public DorisBinaryComparisonOperator getOp() { + return (DorisBinaryComparisonOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftExpectedValue = getLeftExpression().getExpectedValue(); + DorisConstant rightExpectedValue = getRightExpression().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + + public enum DorisBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return left.valueEquals(right); + } + }, + NOT_EQUALS("!=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean()) { + return DorisConstant.createBooleanConstant(!valueEquals.asBoolean()); + } + // maybe DorisNULLConstant or null object + return valueEquals; + } + }, + LESS("<") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return left.valueLessThan(right); + } + }, + LESS_EQUALS("<=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return valueEquals; + } + return valueLessThan; + } + }, + GREATER(">") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } + if (valueLessThan.isNull()) { + return valueLessThan; + } + return DorisConstant.createBooleanConstant(!valueLessThan.asBoolean()); + } + }, + GREATER_EQUALS(">=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + if (valueLessThan.isNull()) { + return valueLessThan; + } + return DorisConstant.createBooleanConstant(!valueLessThan.asBoolean()); + } + }; + + private final String textRepresentation; + + DorisBinaryComparisonOperator(String text) { + textRepresentation = text; + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java new file mode 100644 index 000000000..858c3bf2f --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java @@ -0,0 +1,129 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisBinaryLogicalOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryLogicalOperation(DorisExpression left, DorisExpression right, DorisBinaryLogicalOperator op) { + super(left, right, op); + } + + public DorisExpression getLeftExpr() { + return super.getLeft(); + } + + public DorisExpression getRightExpr() { + return super.getRight(); + } + + public DorisBinaryLogicalOperator getOp() { + return (DorisBinaryLogicalOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = getLeftExpr().getExpectedValue(); + DorisConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + public enum DorisBinaryLogicalOperator implements BinaryOperatorNode.Operator { + /* + * null and false -> false null and true -> null null or false -> null null or true -> true + */ + AND("AND", "and") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant leftVal = left.cast(DorisDataType.BOOLEAN); + DorisConstant rightVal = right.cast(DorisDataType.BOOLEAN); + assert leftVal.isNull() || leftVal.isBoolean() : leftVal + "is not null or boolean"; + assert rightVal.isNull() || rightVal.isBoolean() : rightVal + "is not null or boolean"; + if (leftVal.isNull() && rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (leftVal.isNull()) { + if (!rightVal.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } else { + return DorisConstant.createNullConstant(); + } + } + if (rightVal.isNull()) { + if (!leftVal.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } else { + return DorisConstant.createNullConstant(); + } + } + if (leftVal.asBoolean() && right.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + return DorisConstant.createBooleanConstant(false); + } + }, + OR("OR", "or") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant leftVal = left.cast(DorisDataType.BOOLEAN); + DorisConstant rightVal = right.cast(DorisDataType.BOOLEAN); + assert leftVal.isNull() || leftVal.isBoolean() : leftVal + "is not null or boolean"; + assert rightVal.isNull() || rightVal.isBoolean() : rightVal + "is not null or boolean"; + if (leftVal.isNull() && rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (leftVal.isNull()) { + if (rightVal.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } else { + return DorisConstant.createNullConstant(); + } + } + if (rightVal.isNull()) { + if (leftVal.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } else { + return DorisConstant.createNullConstant(); + } + } + if (leftVal.asBoolean() || right.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + return DorisConstant.createBooleanConstant(false); + } + }; + + private final String[] textRepresentations; + + DorisBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public DorisBinaryLogicalOperator getRandomOp() { + return Randomly.fromOptions(values()); + } + + public static DorisBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryOperation.java b/src/sqlancer/doris/ast/DorisBinaryOperation.java new file mode 100644 index 000000000..ffe80b86d --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DorisBinaryOperation extends NewBinaryOperatorNode implements DorisExpression { + public DorisBinaryOperation(DorisExpression left, DorisExpression right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/doris/ast/DorisCaseOperation.java b/src/sqlancer/doris/ast/DorisCaseOperation.java new file mode 100644 index 000000000..02641408a --- /dev/null +++ b/src/sqlancer/doris/ast/DorisCaseOperation.java @@ -0,0 +1,36 @@ +package sqlancer.doris.ast; + +import java.util.List; + +public class DorisCaseOperation implements DorisExpression { + + private final DorisExpression expr; + private final List conditions; + private final List thenClauses; + private final DorisExpression elseClause; + + public DorisCaseOperation(DorisExpression expr, List conditions, List thenClauses, + DorisExpression elseClause) { + this.expr = expr; + this.conditions = conditions; + this.thenClauses = thenClauses; + this.elseClause = elseClause; + } + + public DorisExpression getExpr() { + return expr; + } + + public List getConditions() { + return conditions; + } + + public List getThenClauses() { + return thenClauses; + } + + public DorisExpression getElseClause() { + return elseClause; + } + +} diff --git a/src/sqlancer/doris/ast/DorisCastOperation.java b/src/sqlancer/doris/ast/DorisCastOperation.java new file mode 100644 index 000000000..e93b8b226 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisCastOperation.java @@ -0,0 +1,45 @@ +package sqlancer.doris.ast; + +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisCastOperation implements DorisExpression { + DorisExpression expr; + DorisDataType type; + + public DorisCastOperation(DorisExpression expr, DorisCompositeDataType type) { + this.expr = expr; + this.type = type.getPrimitiveDataType(); + } + + public DorisCastOperation(DorisExpression expr, DorisDataType type) { + this.expr = expr; + this.type = type; + } + + public DorisExpression getExpr() { + return expr; + } + + public DorisExpression getExpression() { + return expr; + } + + public DorisDataType getType() { + return type; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type); + } + + @Override + public DorisDataType getExpectedType() { + return type; + } +} diff --git a/src/sqlancer/doris/ast/DorisColumnReference.java b/src/sqlancer/doris/ast/DorisColumnReference.java new file mode 100644 index 000000000..ce6aee058 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.doris.DorisSchema; + +public class DorisColumnReference extends ColumnReferenceNode + implements DorisExpression { + public DorisColumnReference(DorisSchema.DorisColumn column) { + super(column); + } +} diff --git a/src/sqlancer/doris/ast/DorisColumnValue.java b/src/sqlancer/doris/ast/DorisColumnValue.java new file mode 100644 index 000000000..fe3ec1448 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisColumnValue.java @@ -0,0 +1,53 @@ +package sqlancer.doris.ast; + +import java.util.Objects; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisColumnValue extends ColumnReferenceNode implements DorisExpression { + + private final DorisConstant expectedValue; + + public DorisColumnValue(DorisColumn column, DorisConstant value) { + super(column); + this.expectedValue = value; + } + + @Override + public DorisConstant getExpectedValue() { + return expectedValue; + } + + @Override + public DorisDataType getExpectedType() { + return getColumn().getType().getPrimitiveDataType(); + } + + public static DorisColumnValue create(DorisColumn column, DorisConstant value) { + return new DorisColumnValue(column, value); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DorisColumnValue that = (DorisColumnValue) o; + if (!this.getColumn().getName().equals(that.getColumn().getName())) { + return false; + } + return Objects.equals(expectedValue, that.expectedValue); + } + + @Override + public int hashCode() { + String nameAndValue = this.getColumn().getName(); + nameAndValue += expectedValue == null ? "NULL" : expectedValue.toString(); + return Objects.hash(nameAndValue); + } +} diff --git a/src/sqlancer/doris/ast/DorisConstant.java b/src/sqlancer/doris/ast/DorisConstant.java new file mode 100644 index 000000000..2ad2500ed --- /dev/null +++ b/src/sqlancer/doris/ast/DorisConstant.java @@ -0,0 +1,680 @@ +package sqlancer.doris.ast; + +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.utils.DorisNumberUtils; + +public abstract class DorisConstant implements DorisExpression { + + private DorisConstant() { + } + + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public boolean isNum() { + // for INT, FLOAT, BOOLEAN + return false; + } + + public boolean isDate() { + return false; + } + + public boolean isDatetime() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + public abstract DorisConstant cast(DorisDataType dataType); + + public abstract DorisConstant valueEquals(DorisConstant rightVal); + + public abstract DorisConstant valueLessThan(DorisConstant rightVal); + + public static class DorisNullConstant extends DorisConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.NULL; + } + } + + public static class DorisIntConstant extends DorisConstant { + + private final long value; + + public DorisIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return this; + case FLOAT: + case DECIMAL: + return new DorisFloatConstant(value); + case VARCHAR: + return new DorisTextConstant(String.valueOf(value)); + case BOOLEAN: + return new DorisBooleanConstant(value != 0); + default: + return DorisConstant.createNullConstant(); + } + } + + @Override + public long asInt() { + return value; + } + + @Override + public boolean asBoolean() { + return value != 0; + } + + @Override + public double asFloat() { + return value; + } + + @Override + public String asString() { + return String.valueOf(value); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isNum()) { + return DorisConstant.createBooleanConstant(value == rightVal.asFloat()); + } + + throw new AssertionError(rightVal); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isNum()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } + + throw new AssertionError(rightVal); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.INT; + } + + } + + public static class DorisFloatConstant extends DorisConstant { + + private final double value; + + public DorisFloatConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public boolean isFloat() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "3.40282347e+38"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-3.40282347e+38"; + } + + return String.valueOf(value); + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return new DorisIntConstant((long) value); + case FLOAT: + case DECIMAL: + return this; + case VARCHAR: + return new DorisTextConstant(String.valueOf(value)); + case BOOLEAN: + return new DorisBooleanConstant(value >= 1); + default: + return null; + } + } + + @Override + public double asFloat() { + return value; + } + + @Override + public String asString() { + return toString(); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DorisConstant.createBooleanConstant(value == rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DorisConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + } + + public static class DorisTextConstant extends DorisConstant { + + private final String value; + + public DorisTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; + } + + @Override + public String asString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + // Currently only supports conversion of int text to int, not float text, see + // https://github.com/apache/doris/issues/18227 + if (DorisNumberUtils.isNumber(value)) { + long val = (long) Double.parseDouble(value); + return new DorisIntConstant(val); + } + return new DorisNullConstant(); + case FLOAT: + case DECIMAL: + if (DorisNumberUtils.isNumber(value)) { + return new DorisFloatConstant(Double.parseDouble(value)); + } + return new DorisNullConstant(); + case DATE: + if (DorisNumberUtils.isDate(value)) { + return new DorisDateConstant(value); + } + return new DorisNullConstant(); + case DATETIME: + if (DorisNumberUtils.isDatetime(value)) { + return new DorisDatetimeConstant(value); + } + return new DorisNullConstant(); + case VARCHAR: + return this; + case BOOLEAN: + if ("false".contentEquals(value.toLowerCase())) { + return new DorisBooleanConstant(false); + } + if ("true".contentEquals(value.toLowerCase())) { + return new DorisBooleanConstant(true); + } + return new DorisNullConstant(); + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isString()) { + return DorisConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } + if (DorisNumberUtils.isNumber(value) && rightVal.isNum()) { + return DorisConstant.createBooleanConstant(Double.parseDouble(value) == rightVal.asFloat()); + } + // Doris currently does not support judgment between string types and other types, such date, datetime + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isString()) { + return DorisConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } + if (DorisNumberUtils.isNumber(value) && rightVal.isNum()) { + return DorisConstant.createBooleanConstant(Double.parseDouble(value) < rightVal.asFloat()); + } + // Doris currently does not support judgment between string types and other types, such date, datetime + return DorisConstant.createBooleanConstant(false); + } + + } + + public static class DorisDateConstant extends DorisConstant { + + public String textRepr; + + public DorisDateConstant(long val) { + textRepr = DorisNumberUtils.timestampToDateText(val); + } + + public DorisDateConstant(String textRepr) { + this.textRepr = textRepr; + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + @Override + public String asString() { + return textRepr; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case VARCHAR: + return new DorisTextConstant(textRepr); + case DATE: + return this; + case DATETIME: + return new DorisDatetimeConstant(DorisNumberUtils.dateTextToDatetimeText(textRepr)); + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && rightVal.asString().contentEquals("CURRENT_TIMESTAMP")) { + return DorisConstant.createBooleanConstant(false); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant.createBooleanConstant(DorisNumberUtils.dateEqual(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && rightVal.asString().contentEquals("CURRENT_TIMESTAMP")) { + return DorisConstant.createBooleanConstant( + DorisNumberUtils.dateLessThan(textRepr, DorisNumberUtils.getCurrentTimeText())); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.dateLessThan(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public boolean isDate() { + return true; + } + } + + public static class DorisDatetimeConstant extends DorisConstant { + + public String textRepr; + + public DorisDatetimeConstant(long val) { + textRepr = DorisNumberUtils.timestampToDatetimeText(val); + } + + public DorisDatetimeConstant(String textRepr) { + this.textRepr = textRepr; + } + + public DorisDatetimeConstant() { + textRepr = "CURRENT_TIMESTAMP"; + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + @Override + public String asString() { + return textRepr; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case VARCHAR: + return new DorisTextConstant(textRepr); + case DATE: + return new DorisDatetimeConstant(DorisNumberUtils.datetimeTextToDateText(textRepr)); + case DATETIME: + return this; + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && (rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + || textRepr.contentEquals("CURRENT_TIMESTAMP"))) { + boolean isEq = rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + && textRepr.contentEquals("CURRENT_TIMESTAMP"); + return DorisConstant.createBooleanConstant(isEq); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.datetimeEqual(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && (rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + || textRepr.contentEquals("CURRENT_TIMESTAMP"))) { + String leftText = textRepr; + String rightText = rightVal.asString(); + if (leftText.contentEquals(rightText)) { + return DorisConstant.createBooleanConstant(false); + } + if (leftText.contentEquals("CURRENT_TIMESTAMP")) { + leftText = DorisNumberUtils.getCurrentTimeText(); + } + if (rightText.contentEquals("CURRENT_TIMESTAMP")) { + rightText = DorisNumberUtils.getCurrentTimeText(); + } + boolean lessThan = DorisNumberUtils.dateLessThan(leftText, rightText); + return DorisConstant.createBooleanConstant(lessThan); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.datetimeLessThan(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public boolean isDatetime() { + return true; + } + + } + + public static class DorisBooleanConstant extends DorisConstant { + + private final boolean value; + + public DorisBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public String asString() { + return toString(); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return new DorisIntConstant(value ? 1 : 0); + case FLOAT: + case DECIMAL: + return new DorisFloatConstant(value ? 1 : 0); + case BOOLEAN: + return this; + case VARCHAR: + return new DorisTextConstant(value ? "1" : "0"); + default: + return null; + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isBoolean()) { + return DorisConstant.createBooleanConstant(value == rightVal.asBoolean()); + } + if (rightVal.isNum() || rightVal.isString() && DorisNumberUtils.isNumber(rightVal.asString())) { + return DorisConstant.createBooleanConstant((value ? 1 : 0) == rightVal.asFloat()); + } + throw new AssertionError(rightVal); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isBoolean()) { + return DorisConstant.createBooleanConstant(value == rightVal.asBoolean()); + } + if (rightVal.isNum() || rightVal.isString() && DorisNumberUtils.isNumber(rightVal.asString())) { + return DorisConstant.createBooleanConstant((value ? 1 : 0) == rightVal.asFloat()); + } + throw new AssertionError(rightVal); + } + } + + public static DorisConstant createStringConstant(String text) { + return new DorisTextConstant(text); + } + + public static DorisConstant createFloatConstant(double val) { + return new DorisFloatConstant(val); + } + + public static DorisConstant createIntConstant(long val) { + return new DorisIntConstant(val); + } + + public static DorisConstant createNullConstant() { + return new DorisNullConstant(); + } + + public static DorisConstant createBooleanConstant(boolean val) { + return new DorisBooleanConstant(val); + } + + public static DorisConstant createDateConstant(long integer) { + return new DorisDateConstant(integer); + } + + public static DorisConstant createDateConstant(String date) { + return new DorisDateConstant(date); + } + + public static DorisConstant createDatetimeConstant(long integer) { + return new DorisDatetimeConstant(integer); + } + + public static DorisConstant createDatetimeConstant(String datetime) { + return new DorisDatetimeConstant(datetime); + } + + public static DorisConstant createDatetimeConstant() { + // use CURRENT_TIMESTAMP + return new DorisDatetimeConstant(); + } + +} diff --git a/src/sqlancer/doris/ast/DorisExpression.java b/src/sqlancer/doris/ast/DorisExpression.java new file mode 100644 index 000000000..ee5cc7a26 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisExpression.java @@ -0,0 +1,15 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; + +public interface DorisExpression extends Expression { + default DorisSchema.DorisDataType getExpectedType() { + return null; + } + + default DorisConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/doris/ast/DorisFunction.java b/src/sqlancer/doris/ast/DorisFunction.java new file mode 100644 index 000000000..1f816a7e3 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisFunction.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DorisFunction extends NewFunctionNode implements DorisExpression { + public DorisFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/doris/ast/DorisFunctionOperation.java b/src/sqlancer/doris/ast/DorisFunctionOperation.java new file mode 100644 index 000000000..05f5370b1 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisFunctionOperation.java @@ -0,0 +1,281 @@ +package sqlancer.doris.ast; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.gen.DorisNewExpressionGenerator; + +public class DorisFunctionOperation implements DorisExpression { + + private DorisFunction function; + private List args; + + // https://doris.apache.org/zh-CN/docs/dev/summary/basic-summary + public enum DorisFunction { + + // Array functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/array-functions/array + // Skip now + + // Date functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/date-time-functions/convert_tz/ + CONVERT_TZ(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CURDATE(false, DorisDataType.DATE), CURRENT_DATE(false, DorisDataType.DATE), + CURTIME(false, DorisDataType.VARCHAR), CURRENT_TIME(false, DorisDataType.VARCHAR), + CURRENT_TIMESTAMP(false, DorisDataType.DATETIME), LOCALTIME(false, DorisDataType.DATETIME), + LOCALTIMESTAMP(false, DorisDataType.DATETIME), NOW(false, DorisDataType.DATETIME), + YEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + QUARTER(false, DorisDataType.INT, DorisDataType.DATETIME), + MONTH(false, DorisDataType.INT, DorisDataType.DATETIME), DAY(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFYEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFMONTH(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFWEEK(false, DorisDataType.INT, DorisDataType.DATETIME), WEEK(false, DorisDataType.INT, DorisDataType.DATE), + WEEKDAY(false, DorisDataType.INT, DorisDataType.DATE), + WEEKOFYEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + YEARWEEK(false, DorisDataType.INT, DorisDataType.DATE), + DAYNAME(false, DorisDataType.VARCHAR, DorisDataType.DATETIME), + MONTHNAME(false, DorisDataType.VARCHAR, DorisDataType.DATETIME), + HOUR(false, DorisDataType.INT, DorisDataType.DATETIME), + MINUTE(false, DorisDataType.INT, DorisDataType.DATETIME), + SECOND(false, DorisDataType.INT, DorisDataType.DATETIME), + FROM_DAYS(false, DorisDataType.DATE, DorisDataType.INT), + LAST_DAYS(false, DorisDataType.DATE, DorisDataType.DATETIME), + TO_MONDAY(false, DorisDataType.DATE, DorisDataType.DATETIME), + FROM_UNIXTIME(false, DorisDataType.DATETIME, DorisDataType.INT), + UNIX_TIMESTAMP(false, DorisDataType.INT, DorisDataType.DATETIME), UTC_TIMESTAMP(false, DorisDataType.DATETIME), + TO_DATE(false, DorisDataType.DATE, DorisDataType.DATETIME), + TO_DAYS(false, DorisDataType.INT, DorisDataType.DATETIME), + TIME_TO_SEC(false, DorisDataType.INT, DorisDataType.DATETIME), + // EXTRACT(1), // select extract(year from '2022-09-22 17:01:30') as year, currently not considered + MAKEDATE(false, DorisDataType.DATE, DorisDataType.INT, DorisDataType.INT), + STR_TO_DATE(false, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + TIME_ROUND(false, DorisDataType.DATETIME, DorisDataType.DATETIME), + TIME_DIFF(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.DATETIME), + TIMESTAMPADD(false, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.DATETIME), + TIMESTAMPDIFF(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.DATETIME), + DATE_ADD(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_TRUNC(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_FORMAT(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATEDIFF(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.DATETIME), + // MICROSECONDS_ADD(false), + MINUTES_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MINUTES_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + MINUTES_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + SECONDS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + SECONDS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + SECONDS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + HOURS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + HOURS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + HOURS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + DAYS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + DAYS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + DAYS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + WEEKS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + WEEKS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + WEEKS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MONTHS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MONTHS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + MONTHS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + YEARS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + YEARS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + YEARS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + + // GIS functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/spatial-functions/st_x + // Skip now + + // String functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/string-functions/to_base64 + TO_BASE64(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + FROM_BASE64(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ASCII(false, DorisDataType.INT, DorisDataType.VARCHAR), LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + BIT_LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + CHAR_LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + LPAD(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + RPAD(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + LOWER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LCASE(false, DorisDataType.INT, DorisDataType.VARCHAR), + UPPER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UCASE(false, DorisDataType.INT, DorisDataType.VARCHAR), + INITCAP(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + REPEAT(false, DorisDataType.VARCHAR, DorisDataType.INT), + REVERSE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CHAR(true, DorisDataType.VARCHAR, DorisDataType.INT), + CONCAT(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CONCAT_WS(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + SUBSTR(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.INT), + SUBSTRING(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + SUB_REPLACE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + APPEND_TRAILING_CHAR_IF_ABSENT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ENDS_WITH(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + STARTS_WITH(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + TRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LTRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + RTRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NULL_OR_EMPTY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR), + NOT_NULL_OR_EMPTY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR), + HEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UNHEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ELT(true, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + INSTR(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LOCATE(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + // FIELD(1, true), + FIND_IN_SET(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + REPLACE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LEFT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + RIGHT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + STRLEFT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + STRRIGHT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + SPLIT_PART(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + // SPLIT_BY_STRING(2), + SUBSTRING_INDEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + MONEY_FORMAT(false, DorisDataType.VARCHAR, DorisDataType.DECIMAL), + PARSE_URL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CONVERT_TO(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + EXTRACT_URL_PARAMETER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UUID(false, DorisDataType.VARCHAR), SPACE(false, DorisDataType.VARCHAR, DorisDataType.INT), + // SLEEP(1), + ESQUERY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK_FIRST_N(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK_LAST_N(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + // MULTI_SEARCH_ALL_POSITIONS(2), + // MULTI_MATCH_ANY(2), + + // BITMAP functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/bitmap-functions/to_bitmap + // skip now + + // Bitwise functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/bitwise-functions/bitand + BITAND(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITOR(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITXOR(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITNOT(false, DorisDataType.INT, DorisDataType.INT), + + // condition funtions + // case(), + COALESCE(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + IF(false, DorisDataType.VARCHAR, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + IFNULL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NVL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NULLIF(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + + // JSON Functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/json-functions/jsonb_parse + // skip now + + // Hash functions, + // https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/hash-functions/murmur_hash3_32 + MURMUR_HASH3_32(true, DorisDataType.INT, DorisDataType.VARCHAR), + MURMUR_HASH3_64(true, DorisDataType.INT, DorisDataType.VARCHAR), + + // HLL functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/hll-functions/hll_cardinality + // skip now + + // Math functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/math-functions/conv + CONV(false, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BIN(false, DorisDataType.VARCHAR, DorisDataType.INT), SIN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + COS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), TAN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + ASIN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), ACOS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + ATAN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), E(false, DorisDataType.FLOAT), + PI(false, DorisDataType.FLOAT), EXP(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG2(false, DorisDataType.FLOAT, DorisDataType.FLOAT), LN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG10(false, DorisDataType.FLOAT, DorisDataType.FLOAT), CEIL(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + FLOOR(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + PMOD(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + ROUND(false, DorisDataType.INT, DorisDataType.FLOAT), + ROUND_BANKERS(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.INT), + TRUNCATE(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.INT), + ABS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), SQRT(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + CBRT(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + POW(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + DEGREES(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + RADIANS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), SIGN(false, DorisDataType.INT, DorisDataType.FLOAT), + POSTIVE(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + NEGATIVE(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + GREATEST(true, DorisDataType.FLOAT, DorisDataType.FLOAT), LEAST(true, DorisDataType.FLOAT, DorisDataType.FLOAT), + RANDOM(false, DorisDataType.FLOAT), MOD(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT); + + // encrypt-digest-functions, + // https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/encrypt-digest-functions/aes + // skip now + + private boolean isVariadic; // If isVALid is true, then treat the last argumentTypes as an infinite type + private DorisDataType returnType; + private DorisDataType[] argumentTypes; + private String functionName; + + DorisFunction(String functionName, boolean isVariadic, DorisDataType returnType, + DorisDataType... argumentTypes) { + this.functionName = functionName; + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + DorisFunction(boolean isVariadic, DorisDataType returnType, DorisDataType... argumentTypes) { + this.functionName = toString(); + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + DorisFunction(boolean isVariadic, DorisDataType returnType) { + this.functionName = toString(); + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = null; + } + + public String getFunctionName() { + return functionName; + } + + public static DorisFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public boolean isVariadic() { + return isVariadic; + } + + public boolean isCompatibleWithReturnType(DorisDataType returnType) { + return this.returnType == returnType; + } + + public DorisDataType[] getArgumentTypes() { + if (argumentTypes == null) { + return null; + } + return argumentTypes.clone(); + } + + public DorisFunctionOperation getCall(DorisDataType returnType, DorisNewExpressionGenerator gen, int depth) { + List arguments = new ArrayList<>(); + if (getArgumentTypes() != null) { + Stream.of(getArgumentTypes()).forEach(arg -> arguments.add(gen.generateExpression(arg, depth + 1))); + } + return new DorisFunctionOperation(this, arguments); + } + + public static List getFunctionsCompatibleWith(DorisDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)) + .collect(Collectors.toList()); + } + + } + + public DorisFunctionOperation(DorisFunction function, List args) { + this.function = function; + this.args = args; + } + + public List getArgs() { + return args; + } + + public DorisFunction getFunction() { + return function; + } + +} diff --git a/src/sqlancer/doris/ast/DorisInOperation.java b/src/sqlancer/doris/ast/DorisInOperation.java new file mode 100644 index 000000000..96ccd0998 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisInOperation.java @@ -0,0 +1,52 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.doris.DorisSchema; + +public class DorisInOperation extends NewInOperatorNode implements DorisExpression { + + private final DorisExpression leftExpr; + private final List rightExpr; + + public DorisInOperation(DorisExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + this.leftExpr = left; + this.rightExpr = right; + } + + @Override + public DorisSchema.DorisDataType getExpectedType() { + return DorisSchema.DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = leftExpr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean containNull = false; + for (DorisExpression expr : rightExpr) { + DorisConstant rightValue = expr.getExpectedValue(); + if (rightValue == null) { + return null; + } + if (rightValue.isNull()) { + containNull = true; + } else if (rightValue.valueEquals(leftValue).isBoolean() && rightValue.valueEquals(leftValue).asBoolean()) { + return DorisConstant.createBooleanConstant(!isNegated()); + } + } + + if (containNull) { + return DorisConstant.createNullConstant(); + } + // should return false when not considering isNegated op + return DorisConstant.createBooleanConstant(isNegated()); + } +} diff --git a/src/sqlancer/doris/ast/DorisJoin.java b/src/sqlancer/doris/ast/DorisJoin.java new file mode 100644 index 000000000..c92555ec3 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisJoin.java @@ -0,0 +1,109 @@ +package sqlancer.doris.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.gen.DorisNewExpressionGenerator; + +public class DorisJoin implements DorisExpression, Join { + + private final DorisTableReference leftTable; + private final DorisTableReference rightTable; + private final JoinType joinType; + private DorisExpression onCondition; + + public enum JoinType { + INNER, STRAIGHT, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public DorisJoin(DorisTableReference leftTable, DorisTableReference rightTable, JoinType joinType, + DorisExpression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public DorisTableReference getLeftTable() { + return leftTable; + } + + public DorisTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public DorisExpression getOnCondition() { + return onCondition; + } + + public static List getJoins(List tableList, DorisGlobalState globalState) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + DorisTableReference leftTable = tableList.remove(0); + DorisTableReference rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + DorisNewExpressionGenerator joinGen = new DorisNewExpressionGenerator(globalState).setColumns(columns); + switch (DorisJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(DorisJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + break; + case STRAIGHT: + joinExpressions.add(DorisJoin.createStraightJoin(leftTable, rightTable, + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + break; + case LEFT: + joinExpressions.add(DorisJoin.createLeftOuterJoin(leftTable, rightTable, + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + break; + case RIGHT: + joinExpressions.add(DorisJoin.createRightOuterJoin(leftTable, rightTable, + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DorisJoin createInnerJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { + return new DorisJoin(left, right, JoinType.INNER, predicate); + } + + public static DorisJoin createStraightJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { + return new DorisJoin(left, right, JoinType.STRAIGHT, predicate); + } + + public static DorisJoin createRightOuterJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { + return new DorisJoin(left, right, JoinType.RIGHT, predicate); + } + + public static DorisJoin createLeftOuterJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { + return new DorisJoin(left, right, JoinType.LEFT, predicate); + } + + @Override + public void setOnClause(DorisExpression onClause) { + onCondition = onClause; + } +} diff --git a/src/sqlancer/doris/ast/DorisLikeOperation.java b/src/sqlancer/doris/ast/DorisLikeOperation.java new file mode 100644 index 000000000..128a9299d --- /dev/null +++ b/src/sqlancer/doris/ast/DorisLikeOperation.java @@ -0,0 +1,84 @@ +package sqlancer.doris.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisLikeOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisLikeOperation(DorisExpression left, DorisExpression right, DorisLikeOperator op) { + super(left, right, op); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + public DorisExpression getLeftExpr() { + return super.getLeft(); + } + + public DorisExpression getRightExpr() { + return super.getRight(); + } + + public DorisLikeOperator getOp() { + return (DorisLikeOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftVal = getLeftExpr().getExpectedValue(); + DorisConstant rightVal = getRightExpr().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + return getOp().apply(leftVal, rightVal); + } + + public enum DorisLikeOperator implements BinaryOperatorNode.Operator { + LIKE_OPERATOR("LIKE", "like") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (left == null || right == null) { + return null; + } + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean result = LikeImplementationHelper.match(left.asString(), right.asString(), 0, 0, true); + return DorisConstant.createBooleanConstant(result); + } + }, + NOT_LIKE("NOT LIKE", "not like") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (left == null || right == null) { + return null; + } + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean result = LikeImplementationHelper.match(left.asString(), right.asString(), 0, 0, true); + return DorisConstant.createBooleanConstant(!result); + } + }; + + private final String[] textRepresentations; + + DorisLikeOperator(String... text) { + textRepresentations = text.clone(); + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + @Override + public String getTextRepresentation() { + return " " + Randomly.fromOptions(textRepresentations) + " "; + } + } + +} diff --git a/src/sqlancer/doris/ast/DorisOrderByTerm.java b/src/sqlancer/doris/ast/DorisOrderByTerm.java new file mode 100644 index 000000000..01a7ab6d2 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisOrderByTerm.java @@ -0,0 +1,9 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class DorisOrderByTerm extends NewOrderingTerm implements DorisExpression { + public DorisOrderByTerm(DorisExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/doris/ast/DorisPostfixText.java b/src/sqlancer/doris/ast/DorisPostfixText.java new file mode 100644 index 000000000..889f4700b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisPostfixText.java @@ -0,0 +1,9 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DorisPostfixText extends NewPostfixTextNode implements DorisExpression { + public DorisPostfixText(DorisExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/doris/ast/DorisSelect.java b/src/sqlancer/doris/ast/DorisSelect.java new file mode 100644 index 000000000..4921e0688 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisSelect.java @@ -0,0 +1,64 @@ +package sqlancer.doris.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisSelect extends SelectBase + implements DorisExpression, Select { + + public enum DorisSelectDistinctType { + + ALL, DISTINCT, DISTINCTROW, NULL; + + public static DorisSelectDistinctType getRandomWithoutNull() { + DorisSelectDistinctType sft; + do { + sft = Randomly.fromOptions(values()); + } while (sft == DorisSelectDistinctType.NULL); + return sft; + } + } + + private DorisSelectDistinctType selectDistinctType = DorisSelectDistinctType.ALL; + + public void setDistinct(boolean isDistinct) { + if (isDistinct) { + this.selectDistinctType = DorisSelectDistinctType.DISTINCT; + } else { + this.selectDistinctType = DorisSelectDistinctType.ALL; + } + } + + public void setDistinct(DorisSelectDistinctType type) { + this.selectDistinctType = type; + } + + public boolean isDistinct() { + return this.selectDistinctType == DorisSelectDistinctType.DISTINCT + || this.selectDistinctType == DorisSelectDistinctType.DISTINCTROW; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DorisExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DorisJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DorisToStringVisitor.asString(this); + } +} diff --git a/src/sqlancer/doris/ast/DorisTableReference.java b/src/sqlancer/doris/ast/DorisTableReference.java new file mode 100644 index 000000000..b8a2cc2a2 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisSchema; + +public class DorisTableReference extends TableReferenceNode + implements DorisExpression { + public DorisTableReference(DorisSchema.DorisTable table) { + super(table); + } +} diff --git a/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java new file mode 100644 index 000000000..f6961c7b7 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java @@ -0,0 +1,86 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DorisExpression { + + public DorisUnaryPostfixOperation(DorisExpression expr, DorisUnaryPostfixOperator op) { + super(expr, op); + } + + public DorisExpression getExpression() { + return getExpr(); + } + + public DorisUnaryPostfixOperator getOp() { + return (DorisUnaryPostfixOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } + + public enum DorisUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public DorisDataType[] getInputDataTypes() { + return DorisDataType.values(); + } + + @Override + public DorisConstant apply(DorisConstant value) { + return DorisConstant.createBooleanConstant(value.isNull()); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public DorisDataType[] getInputDataTypes() { + return DorisDataType.values(); + } + + @Override + public DorisConstant apply(DorisConstant value) { + return DorisConstant.createBooleanConstant(!value.isNull()); + } + }; + + private final String textRepresentations; + + DorisUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static DorisUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract DorisDataType[] getInputDataTypes(); + + public abstract DorisConstant apply(DorisConstant value); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +} diff --git a/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java new file mode 100644 index 000000000..ffd919fdc --- /dev/null +++ b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java @@ -0,0 +1,111 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements DorisExpression { + + public DorisUnaryPrefixOperation(DorisExpression expr, DorisUnaryPrefixOperator op) { + super(expr, op); + } + + public DorisExpression getExpression() { + return getExpr(); + } + + public DorisUnaryPrefixOperator getOp() { + return (DorisUnaryPrefixOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return getOp().getExpressionType(getExpression()); + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } + + public enum DorisUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", DorisDataType.BOOLEAN, DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return DorisDataType.BOOLEAN; + } + + @Override + protected DorisConstant apply(DorisConstant value) { + if (value.isNull()) { + return DorisConstant.createNullConstant(); + } else { + return DorisConstant.createBooleanConstant(!value.cast(DorisDataType.BOOLEAN).asBoolean()); + } + } + }, + + UNARY_PLUS("+", DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return expr.getExpectedType(); + } + + @Override + protected DorisConstant apply(DorisConstant value) { + return value; + } + }, + UNARY_MINUS("-", DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return expr.getExpectedType(); + } + + @Override + protected DorisConstant apply(DorisConstant value) { + if (value.isNull()) { + return DorisConstant.createNullConstant(); + } + try { + if (value.isInt()) { + return DorisConstant.createIntConstant(-value.asInt()); + } + if (value.isFloat()) { + return DorisConstant.createFloatConstant(-value.asFloat()); + } + return null; + } catch (UnsupportedOperationException e) { + return null; + } + } + }; + + private String textRepresentation; + private DorisDataType[] dataTypes; + + DorisUnaryPrefixOperator(String textRepresentation, DorisDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract DorisDataType getExpressionType(DorisExpression expr); + + public DorisDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + protected abstract DorisConstant apply(DorisConstant value); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} diff --git a/src/sqlancer/doris/gen/DorisAlterTableGenerator.java b/src/sqlancer/doris/gen/DorisAlterTableGenerator.java new file mode 100644 index 000000000..bfac074fc --- /dev/null +++ b/src/sqlancer/doris/gen/DorisAlterTableGenerator.java @@ -0,0 +1,50 @@ +package sqlancer.doris.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisTable; + +public final class DorisAlterTableGenerator { + + private DorisAlterTableGenerator() { + } + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(DorisCompositeDataType.getRandomWithoutNull().toString()); + break; + case ALTER_COLUMN: + sb.append("MODIFY COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" "); + sb.append(DorisCompositeDataType.getRandomWithoutNull().toString()); + break; + case DROP_COLUMN: + sb.append("DROP COLUMN "); + sb.append(table.getRandomColumn().getName()); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java new file mode 100644 index 000000000..b155e0381 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -0,0 +1,35 @@ +package sqlancer.doris.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisDeleteGenerator extends AbstractDeleteGenerator { + + private final DorisGlobalState globalState; + + private DorisDeleteGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(DorisGlobalState globalState) { + return new DorisDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + appendDeleteFromTable(table.getName()); + if (Randomly.getBoolean()) { + appendWhereClause(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + DorisErrors.addExpressionErrors(errors); + } + } + +} diff --git a/src/sqlancer/doris/gen/DorisDropTableGenerator.java b/src/sqlancer/doris/gen/DorisDropTableGenerator.java new file mode 100644 index 000000000..c8bbc67d4 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDropTableGenerator.java @@ -0,0 +1,28 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; + +public final class DorisDropTableGenerator { + + private DorisDropTableGenerator() { + } + + public static SQLQueryAdapter dropTable(DorisGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); + if (Randomly.getBoolean()) { + sb.append(" FORCE "); + } + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisDropViewGenerator.java b/src/sqlancer/doris/gen/DorisDropViewGenerator.java new file mode 100644 index 000000000..4f87ba88d --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDropViewGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; + +public final class DorisDropViewGenerator { + + private DorisDropViewGenerator() { + } + + public static SQLQueryAdapter dropView(DorisGlobalState globalState) { + if (globalState.getSchema().getTables(t -> t.isView()).isEmpty()) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP VIEW "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + // TODO: DROP VIEW syntax: DROP MATERIALIZED VIEW [IF EXISTS] mv_name ON table_name; + // should record original table name in view table + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisIndexGenerator.java b/src/sqlancer/doris/gen/DorisIndexGenerator.java new file mode 100644 index 000000000..308c09c3f --- /dev/null +++ b/src/sqlancer/doris/gen/DorisIndexGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.doris.gen; + +import java.sql.SQLException; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; + +public class DorisIndexGenerator extends AbstractIndexGenerator { + + private final DorisGlobalState globalState; + + public DorisIndexGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } + return new DorisIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + DorisTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); + appendCreateIndex(false); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(globalState.getSchema().getFreeIndexName()); + sb.append(" ON "); + sb.append(randomTable.getName()); + // Doris only supports CREATE INDEX on a single column; index type is BITMAP + List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), 1); + appendIndexColumnList(subset, false); + sb.append(" "); + if (Randomly.getBoolean()) { + sb.append("USING BITMAP "); + } + } + +} diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java new file mode 100644 index 000000000..e05bc85fd --- /dev/null +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -0,0 +1,45 @@ +package sqlancer.doris.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisInsertGenerator extends AbstractInsertGenerator { + + private final DorisGlobalState globalState; + + public DorisInsertGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + return new DorisInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + buildInsertInto(table.getName(), columns); + DorisErrors.addInsertErrors(errors); + } + + @Override + protected void insertValue(DorisColumn column) { + if (column.hasDefaultValue() && Randomly.getBooleanWithRatherLowProbability()) { + sb.append("DEFAULT"); + } else { + String value = DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 + sb.append(value); + } + } + +} diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java new file mode 100644 index 000000000..bddc6cc5a --- /dev/null +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -0,0 +1,545 @@ +package sqlancer.doris.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.doris.DorisBugs; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.DorisSchema.DorisRowValue; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.ast.DorisAggregateOperation; +import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; +import sqlancer.doris.ast.DorisBetweenOperation; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation.DorisBinaryArithmeticOperator; +import sqlancer.doris.ast.DorisBinaryComparisonOperation; +import sqlancer.doris.ast.DorisBinaryComparisonOperation.DorisBinaryComparisonOperator; +import sqlancer.doris.ast.DorisBinaryLogicalOperation; +import sqlancer.doris.ast.DorisBinaryLogicalOperation.DorisBinaryLogicalOperator; +import sqlancer.doris.ast.DorisCaseOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisColumnReference; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunctionOperation.DorisFunction; +import sqlancer.doris.ast.DorisInOperation; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisLikeOperation; +import sqlancer.doris.ast.DorisOrderByTerm; +import sqlancer.doris.ast.DorisPostfixText; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisNewExpressionGenerator extends TypedExpressionGenerator + implements NoRECGenerator, + TLPWhereGenerator { + + private final DorisGlobalState globalState; + private List tables; + + private final int maxDepth; + private boolean allowAggregateFunctions; + private DorisRowValue rowValue; + + private Set columnOfLeafNode; + + public DorisNewExpressionGenerator setRowValue(DorisRowValue rowValue) { + this.rowValue = rowValue; + return this; + } + + public void setColumnOfLeafNode(Set columnOfLeafNode) { + this.columnOfLeafNode = columnOfLeafNode; + } + + public DorisNewExpressionGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + @Override + public DorisExpression generateLeafNode(DorisDataType dataType) { + if (Randomly.getBoolean()) { + return generateConstant(dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(dataType); + } else { + return createColumnOfType(dataType); + } + } + } + + final List filterColumns(DorisDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + private DorisExpression createColumnOfType(DorisDataType type) { + List columns = filterColumns(type); + DorisColumn column = Randomly.fromList(columns); + DorisConstant value = rowValue == null ? null : rowValue.getValues().get(column); + if (columnOfLeafNode != null) { + columnOfLeafNode.add(DorisColumnValue.create(column, value)); + } + return DorisColumnValue.create(column, value); + } + + public List generateOrderBy() { + List randomColumns = Randomly.subset(columns); + return randomColumns.stream() + .map(c -> new DorisOrderByTerm(new DorisColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) + .collect(Collectors.toList()); + } + + @Override + public DorisExpression generateExpression(DorisDataType type, int depth) { + // todo: case operation should be add into generateExpression + + if (Randomly.getBooleanWithRatherLowProbability() || depth >= maxDepth) { + return generateLeafNode(type); + } + + if (globalState.getDbmsSpecificOptions().testFunctions && Randomly.getBooleanWithRatherLowProbability()) { + List applicableFunctions = DorisFunction.getFunctionsCompatibleWith(type); + if (!applicableFunctions.isEmpty()) { + DorisFunction function = Randomly.fromList(applicableFunctions); + return function.getCall(type, this, depth + 1); + } + } + if (!DorisBugs.bug36070 && type != DorisDataType.NULL && globalState.getDbmsSpecificOptions().testCasts + && Randomly.getBooleanWithRatherLowProbability()) { + return new DorisCastOperation(generateExpression(getRandomType(), depth + 1), type); + } + if (!DorisBugs.bug36070 && globalState.getDbmsSpecificOptions().testCase + && Randomly.getBooleanWithRatherLowProbability()) { + DorisExpression expr = generateExpression(DorisDataType.BOOLEAN, depth + 1); + List conditions = new ArrayList<>(); + List cases = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + conditions.add(generateExpression(DorisDataType.BOOLEAN, depth + 1)); + cases.add(generateExpression(type, depth + 1)); + } + DorisExpression elseExpr = null; + if (Randomly.getBoolean()) { + elseExpr = generateExpression(type, depth + 1); + } + return new DorisCaseOperation(expr, conditions, cases, elseExpr); + } + + switch (type) { + case INT: + return generateIntExpression(depth); + case BOOLEAN: + return generateBooleanExpression(depth); + case FLOAT: + case DECIMAL: + case DATE: + case DATETIME: + case VARCHAR: + case NULL: + return generateConstant(type); + default: + throw new AssertionError(); + } + } + + public List generateExpressions(int nr, DorisDataType type) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(type)); + } + return expressions; + } + + private enum IntExpression { + UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION + } + + private DorisExpression generateIntExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } + IntExpression intExpression = Randomly.fromOptions(IntExpression.values()); + switch (intExpression) { + case UNARY_OPERATION: + return new DorisUnaryPrefixOperation(generateExpression(DorisDataType.INT, depth + 1), + Randomly.getBoolean() ? DorisUnaryPrefixOperator.UNARY_PLUS : DorisUnaryPrefixOperator.UNARY_MINUS); + case BINARY_ARITHMETIC_OPERATION: + return new DorisBinaryArithmeticOperation(generateExpression(DorisDataType.INT, depth + 1), + generateExpression(DorisDataType.INT, depth + 1), + Randomly.fromOptions(DorisBinaryArithmeticOperator.values())); + default: + throw new AssertionError(); + } + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; + // SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; + } + + DorisExpression generateBooleanExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (DorisBugs.bug36346 || !globalState.getDbmsSpecificOptions().testIn) { + validOptions.remove(BooleanExpression.IN_OPERATION); + } + if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { + validOptions.remove(BooleanExpression.BINARY_LOGICAL_OPERATOR); + } + if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { + validOptions.remove(BooleanExpression.BINARY_COMPARISON); + } + if (DorisBugs.bug36070 || !globalState.getDbmsSpecificOptions().testBetween) { + validOptions.remove(BooleanExpression.BETWEEN); + } + + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + return getPostfix(depth + 1); + case NOT: + return getNOT(depth + 1); + case BETWEEN: + return getBetween(depth + 1); + case IN_OPERATION: + return getIn(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return getBinaryLogical(depth + 1, DorisDataType.BOOLEAN); + case BINARY_COMPARISON: + return getComparison(depth + 1); + case LIKE: + return getLike(depth + 1, DorisDataType.VARCHAR); + default: + throw new AssertionError(); + } + + } + + DorisExpression getPostfix(int depth) { + DorisUnaryPostfixOperator randomOp = DorisUnaryPostfixOperator.getRandom(); + return new DorisUnaryPostfixOperation( + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp); + } + + DorisExpression getNOT(int depth) { + DorisUnaryPrefixOperator op = DorisUnaryPrefixOperator.NOT; + return new DorisUnaryPrefixOperation(generateExpression(op.getRandomInputDataTypes(), depth), op); + } + + DorisExpression getBetween(int depth) { + DorisDataType dataType = Randomly.fromList(Arrays.asList(DorisDataType.values()).stream() + .filter(t -> t != DorisDataType.BOOLEAN).collect(Collectors.toList())); + + return new DorisBetweenOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + generateExpression(dataType, depth), Randomly.getBoolean()); + } + + DorisExpression getIn(int depth) { + DorisDataType dataType = Randomly.fromOptions(DorisDataType.values()); + DorisExpression leftExpr = generateExpression(dataType, depth); + List rightExprs = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + rightExprs.add(generateExpression(dataType, depth)); + } + return new DorisInOperation(leftExpr, rightExprs, Randomly.getBoolean()); + } + + DorisExpression getBinaryLogical(int depth, DorisDataType dataType) { + DorisExpression expr = generateExpression(dataType, depth); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + expr = new DorisBinaryLogicalOperation(expr, generateExpression(DorisDataType.BOOLEAN, depth), + DorisBinaryLogicalOperator.getRandom()); + } + return expr; + } + + DorisExpression getComparison(int depth) { + // 跳过boolean + DorisDataType dataType = Randomly.fromList(Arrays.asList(DorisDataType.values()).stream() + .filter(t -> t != DorisDataType.BOOLEAN).collect(Collectors.toList())); + DorisExpression leftExpr = generateExpression(dataType, depth); + DorisExpression rightExpr = generateExpression(dataType, depth); + return new DorisBinaryComparisonOperation(leftExpr, rightExpr, + Randomly.fromOptions(DorisBinaryComparisonOperator.values())); + } + + DorisExpression getLike(int depth, DorisDataType dataType) { + return new DorisLikeOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + DorisLikeOperation.DorisLikeOperator.LIKE_OPERATOR); + } + + public DorisExpression generateExpressionWithExpectedResult(DorisDataType type) { + DorisExpression expr; + do { + expr = this.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + @Override + public DorisExpression generatePredicate() { + return generateExpression(DorisDataType.BOOLEAN); + } + + @Override + public DorisExpression negatePredicate(DorisExpression predicate) { + return new DorisUnaryPrefixOperation(predicate, DorisUnaryPrefixOperator.NOT); + } + + @Override + public DorisExpression isNull(DorisExpression predicate) { + return new DorisUnaryPostfixOperation(predicate, DorisUnaryPostfixOperator.IS_NULL); + } + + public DorisExpression generateConstant(DorisDataType type, boolean isNullable) { + if (!isNullable) { + return createConstantWithoutNull(type); + } + if (Randomly.getBooleanWithSmallProbability()) { + return createConstant(DorisDataType.NULL); + } + return createConstant(type); + } + + @Override + public DorisExpression generateConstant(DorisDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return DorisConstant.createNullConstant(); + } + return createConstant(type); + } + + public DorisExpression createConstantWithoutNull(DorisDataType type) { + DorisExpression constant = createConstant(type); + int loopCount = 0; + while (constant instanceof DorisConstant.DorisNullConstant && loopCount < 1000) { + constant = createConstant(type); + loopCount++; + } + if (constant instanceof DorisConstant.DorisNullConstant) { + throw new IgnoreMeException(); + } + return constant; + } + + public DorisExpression createConstant(DorisDataType type) { + Randomly r = globalState.getRandomly(); + long timestamp; + switch (type) { + case INT: + if (globalState.getDbmsSpecificOptions().testIntConstants) { + long number = r.getInteger(); + if (DorisBugs.bug36351 && number == -1049190528) { + number = 0; + } + return DorisConstant.createIntConstant(r.getInteger()); + } + return DorisConstant.createNullConstant(); + case BOOLEAN: + if (globalState.getDbmsSpecificOptions().testBooleanConstants) { + return DorisConstant.createBooleanConstant(Randomly.getBoolean()); + } + return DorisConstant.createNullConstant(); + case DECIMAL: + if (globalState.getDbmsSpecificOptions().testDecimalConstants) { + double v = r.getDouble(); + while (v == Double.MAX_VALUE || v == -Double.MAX_VALUE || v == Double.POSITIVE_INFINITY + || v == Double.NEGATIVE_INFINITY) { + v = r.getDouble(); + } + + // e.g. format 1234.413232532 to qualify num 34.4132 + String formatter = "%." + type.getDecimalScale() + "f"; + String vStr = String.format(formatter, v); + int pointPos = vStr.indexOf('.'); + if (pointPos > type.getDecimalPrecision() - type.getDecimalScale()) { + vStr = vStr.substring(pointPos - (type.getDecimalPrecision() - type.getDecimalScale())); + } + return DorisConstant.createFloatConstant(Double.parseDouble(vStr)); + } + return DorisConstant.createNullConstant(); + case FLOAT: + if (globalState.getDbmsSpecificOptions().testFloatConstants) { + return DorisConstant.createFloatConstant((float) r.getDouble()); + } + return DorisConstant.createNullConstant(); + case DATE: + if (globalState.getDbmsSpecificOptions().testDateConstants) { + // [1970-01-01 08:00:00, 3000-01-01 00:00:00] + timestamp = globalState.getRandomly().getLong(0, 32503651200L); + return DorisConstant.createDateConstant(timestamp); + } + return DorisConstant.createNullConstant(); + case DATETIME: + if (globalState.getDbmsSpecificOptions().testDateTimeConstants) { + // [1970-01-01 08:00:00, 3000-01-01 00:00:00] + timestamp = globalState.getRandomly().getLong(0, 32503651200L); + if (DorisBugs.bug36342) { + return DorisConstant.createDatetimeConstant(timestamp); + } + return Randomly.fromOptions(DorisConstant.createDatetimeConstant(timestamp), + DorisConstant.createDatetimeConstant()); + } + return DorisConstant.createNullConstant(); + case VARCHAR: + if (globalState.getDbmsSpecificOptions().testStringConstants) { + String s = r.getString(); + if (s.length() > type.getVarcharLength()) { + s = s.substring(0, type.getVarcharLength()); + } + return DorisConstant.createStringConstant(s); + } + return DorisConstant.createNullConstant(); + case NULL: + return DorisConstant.createNullConstant(); + default: + throw new AssertionError(type); + } + } + + @Override + protected DorisExpression generateColumn(DorisDataType type) { + return null; + } + + @Override + protected DorisDataType getRandomType() { + return Randomly.fromOptions(DorisDataType.values()); + } + + @Override + protected boolean canGenerateColumnOfType(DorisDataType type) { + return false; + } + + public DorisExpression generateArgsForAggregate(DorisAggregateFunction aggregateFunction) { + DorisDataType dataType = Randomly.fromOptions(DorisDataType.values()); + return new DorisAggregateOperation(generateExpressions(aggregateFunction.getNrArgs(), dataType), + aggregateFunction); + } + + public DorisExpression generateAggregate() { + DorisAggregateFunction aggrFunc = DorisAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + public DorisExpression generateHavingClause() { + allowAggregateFunctions = true; + DorisExpression expression = generateExpression(DorisDataType.BOOLEAN); + allowAggregateFunctions = false; + return expression; + } + + public void setAllowAggregateFunctions(boolean allowAggregateFunctions) { + this.allowAggregateFunctions = allowAggregateFunctions; + } + + @Override + public DorisNewExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DorisExpression generateBooleanExpression() { + return generateExpression(DorisDataType.BOOLEAN); + } + + @Override + public DorisSelect generateSelect() { + return new DorisSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DorisTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DorisSelect select, DorisExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + DorisExpression aggr = new DorisAggregateOperation( + List.of(new DorisColumnReference( + new DorisColumn("*", new DorisCompositeDataType(DorisDataType.INT, 0), false, false))), + DorisAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + + } else { + List allColumns = columns.stream().map((c) -> new DorisColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + List constants = new ArrayList<>(); + constants.add(new DorisConstant.DorisIntConstant( + Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByClauses(constants); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DorisSelect select, DorisExpression whereCondition) { + DorisExpression asText = new DorisPostfixText(new DorisCastOperation( + new DorisPostfixText(whereCondition, + " IS NOT NULL AND " + DorisToStringVisitor.asString(whereCondition)), + new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new DorisColumnReference(new DorisColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java new file mode 100644 index 000000000..e1ec50eb2 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -0,0 +1,77 @@ +package sqlancer.doris.gen; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; + +public final class DorisRandomQuerySynthesizer { + + private DorisRandomQuerySynthesizer() { + } + + public static DorisSelect generateSelect(DorisGlobalState globalState, int nrColumns) { + DorisTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + List targetColumns = targetTables.getColumns(); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState).setColumns(targetColumns); + DorisSelect select = new DorisSelect(); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); + int freeColumns = targetColumns.size(); + select.setDistinct(DorisSelect.DorisSelectDistinctType.getRandomWithoutNull()); + List columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + DorisExpression column = null; + if (freeColumns > 0 && Randomly.getBoolean()) { + column = new DorisColumnValue(targetColumns.get(freeColumns - 1), null); + freeColumns -= 1; + columnOfLeafNode.add((DorisColumnValue) column); + } else { + column = gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN); + } + columns.add(column); + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); + } + + List noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean()) { + select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(noExprColumns); + } + if (Randomly.getBoolean()) { + select.setLimitClause(DorisConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(DorisConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + return select; + } + +} diff --git a/src/sqlancer/doris/gen/DorisTableGenerator.java b/src/sqlancer/doris/gen/DorisTableGenerator.java new file mode 100644 index 000000000..74d2f956a --- /dev/null +++ b/src/sqlancer/doris/gen/DorisTableGenerator.java @@ -0,0 +1,112 @@ +package sqlancer.doris.gen; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisTableGenerator { + + // private final ExpectedErrors errors = new ExpectedErrors(); + + public static SQLQueryAdapter createRandomTableStatement(DorisGlobalState globalState) throws SQLException { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return new DorisTableGenerator().getQuery(globalState); + } + + public SQLQueryAdapter getQuery(DorisGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + DorisSchema.DorisTableDataModel dataModel = DorisSchema.DorisTableDataModel.getRandom(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(globalState); + Collections.sort(columns); + if (columns.isEmpty() || !columns.get(0).isKey()) { + return null; // ensure table has at least one key column + } + sb.append(columns.stream().map(DorisColumn::toString).collect(Collectors.joining(", "))); + sb.append(")"); + + List keysColumn = columns.stream().filter(DorisColumn::isKey).collect(Collectors.toList()); + if (globalState.getDbmsSpecificOptions().testDataModel && Randomly.getBoolean() && !keysColumn.isEmpty()) { + sb.append(" " + dataModel).append(" KEY("); + sb.append(keysColumn.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + } + sb.append(generateDistributionStr(globalState, dataModel, keysColumn)); + sb.append(" PROPERTIES (\"replication_num\" = \"1\")"); // now only consider this one parameter + DorisErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + public static String generateDistributionStr(DorisGlobalState globalState, + DorisSchema.DorisTableDataModel dataModel, List keysColumn) { + // DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num] + // DISTRIBUTED BY RANDOM [BUCKETS num] + StringBuilder sb = new StringBuilder(); + sb.append(" DISTRIBUTED BY"); + if (dataModel == DorisSchema.DorisTableDataModel.UNIQUE || Randomly.getBoolean()) { + sb.append(" HASH ("); + sb.append(Randomly.nonEmptySubset(keysColumn).stream().map(DorisColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + } else { + sb.append(" RANDOM"); + } + if (Randomly.getBoolean()) { + sb.append(" BUCKETS ").append(globalState.getRandomly().getInteger(1, 32)); + } + return sb.toString(); + } + + private static List getNewColumns(DorisGlobalState globalState) { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + DorisCompositeDataType columnType = DorisCompositeDataType.getRandomWithoutNull(); + columnType.initColumnArgs(); // set decimalAndVarchar + + boolean iskey = columnType.canBeKey() && Randomly.getBoolean(); + boolean isNullable = Randomly.getBoolean(); + if (!globalState.getDbmsSpecificOptions().testNotNullConstraints) { + isNullable = true; + } + // boolean isHllOrBitmap = (columnType.getPrimitiveDataType() == DorisSchema.DorisDataType.HLL) + // || (columnType.getPrimitiveDataType() == DorisSchema.DorisDataType.BITMAP); + boolean isHllOrBitmap = false; + DorisSchema.DorisColumnAggrType aggrType = DorisSchema.DorisColumnAggrType.NULL; + if (globalState.getDbmsSpecificOptions().testColumnAggr && (isHllOrBitmap || !iskey)) { + aggrType = DorisSchema.DorisColumnAggrType.getRandom(columnType); + } + + boolean hasDefaultValue = globalState.getDbmsSpecificOptions().testDefaultValues && Randomly.getBoolean() + && !isHllOrBitmap; + String defaultValue = ""; + if (hasDefaultValue) { + defaultValue = DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .generateConstant(columnType.getPrimitiveDataType(), isNullable)); + } + columns.add(new DorisColumn(columnName, columnType, iskey, isNullable, aggrType, hasDefaultValue, + defaultValue)); + } + return columns; + } + +} diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java new file mode 100644 index 000000000..93f835eff --- /dev/null +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -0,0 +1,54 @@ +package sqlancer.doris.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisUpdateGenerator extends AbstractUpdateGenerator { + + private final DorisGlobalState globalState; + private DorisNewExpressionGenerator gen; + + private DorisUpdateGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + return new DorisUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new DorisNewExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + appendWhereClause(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + DorisErrors.addInsertErrors(errors); + } + + @Override + protected void updateValue(DorisColumn column) { + if (Randomly.getBooleanWithSmallProbability()) { + DorisExpression expr = gen.generateExpression(column.getType().getPrimitiveDataType()); + sb.append(DorisToStringVisitor.asString(expr)); + } else { + DorisExpression expr = gen.generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()); + sb.append(DorisToStringVisitor.asString(expr)); + } + + } + +} diff --git a/src/sqlancer/doris/gen/DorisViewGenerator.java b/src/sqlancer/doris/gen/DorisViewGenerator.java new file mode 100644 index 000000000..480607378 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisViewGenerator.java @@ -0,0 +1,38 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisViewGenerator { + + private DorisViewGenerator() { + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(DorisToStringVisitor.asString(DorisRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + DorisErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..a7e8b5b02 --- /dev/null +++ b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java @@ -0,0 +1,152 @@ +package sqlancer.doris.oracle; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.DorisSchema.DorisRowValue; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.visitor.DorisExpectedValueVisitor; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public DorisPivotedQuerySynthesisOracle(DorisGlobalState globalState) { + super(globalState); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + protected Query getRectifiedQuery() throws Exception { + DorisTables randomTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); + List columns = randomTables.getColumns(); + DorisSelect selectStatement = new DorisSelect(); + boolean isDistinct = Randomly.getBoolean(); + selectStatement.setDistinct(isDistinct); + pivotRow = randomTables.getRandomRowValue(globalState.getConnection()); + fetchColumns = columns; + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new DorisColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + selectStatement.setFromList( + randomTables.getTables().stream().map(t -> new DorisTableReference(t)).collect(Collectors.toList())); + DorisExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + DorisExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + DorisExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + gen.setColumns(columns); + if (!isDistinct) { + List constants = new ArrayList<>(); + constants.add(new DorisConstant.DorisIntConstant( + Randomly.smallNumber() % selectStatement.getFetchColumns().size() + 1)); + selectStatement.setOrderByClauses(constants); + } + return new SQLQueryAdapter(DorisToStringVisitor.asString(selectStatement), errors); + } + + private DorisExpression generateRectifiedExpression(List columns, DorisRowValue pivotRow) { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState).setColumns(columns); + gen.setRowValue(pivotRow); + DorisExpression expr = gen.generateExpressionWithExpectedResult(DorisDataType.BOOLEAN); + DorisExpression result = null; + if (expr.getExpectedValue().isNull()) { + result = new DorisUnaryPostfixOperation(expr, DorisUnaryPostfixOperation.DorisUnaryPostfixOperator.IS_NULL); + } else if (!expr.getExpectedValue().cast(DorisDataType.BOOLEAN).asBoolean()) { + result = new DorisUnaryPrefixOperation(expr, DorisUnaryPrefixOperation.DorisUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); + sb.append(pivotRowQuery.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (DorisColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL "); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).toString()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + private DorisColumn getFetchValueAliasedColumn(DorisColumn c) { + DorisColumn aliasedColumn = new DorisColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType(), false, false); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + @Override + protected String getExpectedValues(DorisExpression expr) { + return DorisExpectedValueVisitor.asExpectedValues(expr); + } + + private List generateGroupByClause(List columns, DorisRowValue rowValue) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> new DorisColumnValue(c, rowValue.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private DorisExpression generateLimit() { + if (Randomly.getBoolean()) { + return DorisConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private DorisExpression generateOffset() { + if (Randomly.getBoolean()) { + return DorisConstant.createIntConstant(0); + } else { + return null; + } + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..afc1dde43 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -0,0 +1,202 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.ast.DorisAggregateOperation; +import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; +import sqlancer.doris.ast.DorisAlias; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation; +import sqlancer.doris.ast.DorisBinaryOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunction; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningAggregateTester extends DorisQueryPartitioningBase + implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public DorisQueryPartitioningAggregateTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + DorisAggregateFunction aggregateFunction = Randomly.fromOptions(DorisAggregateFunction.MAX, + DorisAggregateFunction.MIN, DorisAggregateFunction.SUM, DorisAggregateFunction.COUNT, + DorisAggregateFunction.AVG); + DorisFunction aggregate = (DorisAggregateOperation) gen + .generateArgsForAggregate(aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add((DorisAggregateOperation) gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByClauses(constants); + } + originalQuery = DorisToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResult == null && secondResult == null) { + return; + } + if (firstResult == null) { + throw new AssertionError(); + } + firstResult = firstResult.replace("\0", ""); + if (firstResult.contentEquals("0") && secondResult == null) { + return; + } + if (secondResult == null) { + throw new AssertionError(); + } + secondResult = secondResult.replace("\0", ""); + if (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(DorisSelect select, DorisFunction aggregate, + List from) { + String metamorphicQuery; + DorisExpression whereClause = gen.generateExpression(DorisDataType.BOOLEAN); + DorisExpression negatedClause = new DorisUnaryPrefixOperation(whereClause, DorisUnaryPrefixOperator.NOT); + DorisExpression notNullClause = new DorisUnaryPostfixOperation(whereClause, DorisUnaryPostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + DorisSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + DorisSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + DorisSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(groupByExpression); + middleSelect.setGroupByExpressions(groupByExpression); + rightSelect.setGroupByExpressions(groupByExpression); + } + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += DorisToStringVisitor.asString(leftSelect) + " UNION ALL " + + DorisToStringVisitor.asString(middleSelect) + " UNION ALL " + + DorisToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List mapped(DorisFunction aggregate) { + + DorisCastOperation count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(Arrays.asList(aggregate)); + case AVG: + DorisFunction sum = new DorisFunction<>(aggregate.getArgs(), + DorisAggregateFunction.SUM); + count = new DorisCastOperation(new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), + new DorisCompositeDataType(DorisDataType.FLOAT, 8)); + return aliasArgs(Arrays.asList(sum, count)); + case STDDEV_POP: + DorisFunction sumSquared = new DorisFunction<>( + Arrays.asList(new DorisBinaryOperation(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DorisBinaryArithmeticOperation.DorisBinaryArithmeticOperator.MULTIPLICATION)), + DorisAggregateFunction.SUM); + count = new DorisCastOperation(new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), + new DorisCompositeDataType(DorisDataType.FLOAT, 8)); + DorisFunction avg = new DorisFunction<>(aggregate.getArgs(), + DorisAggregateFunction.AVG); + return aliasArgs(Arrays.asList(sumSquared, count, avg)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (DorisExpression expr : originalAggregateArgs) { + args.add(new DorisAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(DorisFunction aggregate) { + switch (aggregate.getFunc()) { + case STDDEV_POP: + return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; + case AVG: + return "SUM(agg0::FLOAT)/SUM(agg1)::FLOAT"; + case COUNT: + return DorisAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private DorisSelect getSelect(List aggregates, List from, + DorisExpression whereClause, List joinList) { + DorisSelect leftSelect = new DorisSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinList(joinList); + return leftSelect; + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java new file mode 100644 index 000000000..553e7739a --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -0,0 +1,82 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnReference; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; +import sqlancer.doris.gen.DorisNewExpressionGenerator; + +public class DorisQueryPartitioningBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + DorisSchema s; + DorisTables targetTables; + DorisNewExpressionGenerator gen; + DorisSelect select; + + List groupByExpression; + + public DorisQueryPartitioningBase(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new DorisNewExpressionGenerator(state).setColumns(targetTables.getColumns()); + List allColumnValues = targetTables.getColumns().stream() + .map(c -> new DorisColumnValue(c, null)).collect(Collectors.toList()); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); + initializeTernaryPredicateVariants(); + select = new DorisSelect(); + columnOfLeafNode.addAll(allColumnValues); + groupByExpression = new ArrayList<>(allColumnValues); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List generateFetchColumns() { + List columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new DorisColumnReference(new DorisColumn("*", null, false, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..dc350ea6f --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java @@ -0,0 +1,44 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningDistinctTester extends DorisQueryPartitioningBase { + + public DorisQueryPartitioningDistinctTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + + String unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString + + " UNION ALL " + thirdQueryString + ") tmpTable"; + combinedString.add(unionString); + List secondResultSet = ComparatorHelper.getResultSetFirstColumnAsString(unionString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..97ade18e3 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -0,0 +1,52 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.ast.DorisColumnReference; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningGroupByTester extends DorisQueryPartitioningBase { + + public DorisQueryPartitioningGroupByTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(select.getFetchColumns()); + select.setWhereClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } + + @Override + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java new file mode 100644 index 000000000..434ee01d5 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -0,0 +1,71 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningHavingTester extends DorisQueryPartitioningBase + implements TestOracle { + + public DorisQueryPartitioningHavingTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); + } + select.setFetchColumns(groupByExpression); + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + List constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByClauses(constants); + } + select.setGroupByExpressions(groupByExpression); + select.setHavingClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(predicate); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } + + @Override + protected DorisExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List generateFetchColumns() { + gen.setAllowAggregateFunctions(true); + List expressions = gen.generateExpressions(Randomly.smallNumber() + 1); + gen.setAllowAggregateFunctions(false); + return expressions; + } + +} diff --git a/src/sqlancer/doris/utils/DorisNumberUtils.java b/src/sqlancer/doris/utils/DorisNumberUtils.java new file mode 100644 index 000000000..01dd7238d --- /dev/null +++ b/src/sqlancer/doris/utils/DorisNumberUtils.java @@ -0,0 +1,111 @@ +package sqlancer.doris.utils; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class DorisNumberUtils { + private static Pattern numberPattern = Pattern.compile("-?[0-9]+(\\\\.[0-9]+)?"); + private static Pattern integerPattern = Pattern.compile("^[-\\+]?[\\d]*$"); + private static Pattern datePattern = Pattern + .compile("^([1-9]\\d{3}-)(([0]{0,1}[1-9]-)|([1][0-2]-))(([0-3]{0,1}[0-9]))$"); + private static Pattern datetimePattern = Pattern.compile( + "((([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})-(((0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01]))|((0[469]|11)-(0[1-9]|[12][0-9]|30))|(02-(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|((0[48]|[2468][048]|[3579][26])00))-02-29))\\\\s+([0-1]?[0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])\n"); + + private DorisNumberUtils() { + } + + public static boolean isNumber(String str) { + Matcher m = numberPattern.matcher(str); + return m.matches(); + } + + public static boolean isInteger(String str) { + Matcher m = integerPattern.matcher(str); + return m.matches(); + } + + public static boolean isDate(String str) { + Matcher m = datePattern.matcher(str); + return m.matches(); + } + + public static boolean isDatetime(String str) { + Matcher m = datetimePattern.matcher(str); + return m.matches(); + } + + public static String timestampToDateText(long ts) { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + return dateFormat.format(ts); + } + + public static String timestampToDatetimeText(long ts) { + SimpleDateFormat datetimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return datetimeFormat.format(ts); + } + + public static String dateTextToDatetimeText(String date) { + // '2021-03-12' -> '2021-03-12 00:00:00' + return date + " 00:00:00"; + } + + public static String datetimeTextToDateText(String datetime) { + // '2021-03-12 00:00:00' -> '2021-03-12' + return datetime.substring(0, 10); + } + + public static boolean datetimeEqual(String dt1, String dt2) { + String datetime1 = dt1; + String datetime2 = dt2; + if (isDate(dt1)) { + datetime1 = dateTextToDatetimeText(dt1); + } + if (isDate(dt2)) { + datetime2 = dateTextToDatetimeText(dt2); + } + return datetime1.contentEquals(datetime2); + } + + public static boolean dateEqual(String d1, String d2) { + String date1 = d1; + String date2 = d2; + if (isDatetime(d1)) { + date1 = datetimeTextToDateText(d1); + } + if (isDatetime(d2)) { + date2 = datetimeTextToDateText(d2); + } + return date1.contentEquals(date2); + } + + public static boolean dateLessThan(String d1, String d2) { + String date1 = d1; + String date2 = d2; + if (isDatetime(d1)) { + date1 = datetimeTextToDateText(d1); + } + if (isDatetime(d2)) { + date2 = datetimeTextToDateText(d2); + } + return date1.compareTo(date2) < 0; + } + + public static boolean datetimeLessThan(String dt1, String dt2) { + String datetime1 = dt1; + String datetime2 = dt2; + if (isDate(dt1)) { + datetime1 = dateTextToDatetimeText(dt1); + } + if (isDate(dt2)) { + datetime2 = dateTextToDatetimeText(dt2); + } + return datetime1.compareTo(datetime2) < 0; + } + + public static String getCurrentTimeText() { + SimpleDateFormat datetimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return datetimeFormat.format(new Date()); + } +} diff --git a/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java new file mode 100644 index 000000000..10c90e47e --- /dev/null +++ b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java @@ -0,0 +1,151 @@ +package sqlancer.doris.visitor; + +import java.util.List; + +import sqlancer.doris.ast.DorisAlias; +import sqlancer.doris.ast.DorisBetweenOperation; +import sqlancer.doris.ast.DorisBinaryOperation; +import sqlancer.doris.ast.DorisColumnReference; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunction; +import sqlancer.doris.ast.DorisInOperation; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisOrderByTerm; +import sqlancer.doris.ast.DorisPostfixText; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; + +public class DorisExpectedValueVisitor { + + protected final StringBuilder sb = new StringBuilder(); + + private void print(DorisExpression expr) { + sb.append(DorisToStringVisitor.asString(expr)); + sb.append(" -- "); + sb.append(((DorisExpression) expr).getExpectedValue()); + sb.append("\n"); + } + + public void visit(DorisExpression expr) { + assert expr != null; + if (expr instanceof DorisColumnReference) { + visit((DorisColumnReference) expr); + } else if (expr instanceof DorisUnaryPostfixOperation) { + visit((DorisUnaryPostfixOperation) expr); + } else if (expr instanceof DorisUnaryPrefixOperation) { + visit((DorisUnaryPrefixOperation) expr); + } else if (expr instanceof DorisBinaryOperation) { + visit((DorisBinaryOperation) expr); + } else if (expr instanceof DorisTableReference) { + visit((DorisTableReference) expr); + } else if (expr instanceof DorisFunction) { + visit((DorisFunction) expr); + } else if (expr instanceof DorisBetweenOperation) { + visit((DorisBetweenOperation) expr); + } else if (expr instanceof DorisInOperation) { + visit((DorisInOperation) expr); + } else if (expr instanceof DorisOrderByTerm) { + visit((DorisOrderByTerm) expr); + } else if (expr instanceof DorisAlias) { + visit((DorisAlias) expr); + } else if (expr instanceof DorisPostfixText) { + visit((DorisPostfixText) expr); + } else if (expr instanceof DorisConstant) { + visit((DorisConstant) expr); + } else if (expr instanceof DorisSelect) { + visit((DorisSelect) expr); + } else if (expr instanceof DorisJoin) { + visit((DorisJoin) expr); + } else { + throw new AssertionError(expr); + } + } + + public void visit(DorisColumnReference c) { + print(c); + } + + public void visit(DorisUnaryPostfixOperation op) { + print(op); + visit(op.getExpr()); + } + + public void visit(DorisUnaryPrefixOperation op) { + print(op); + visit(op.getExpr()); + } + + public void visit(DorisBinaryOperation op) { + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(DorisTableReference t) { + print(t); + } + + public void visit(DorisFunction fun) { + print(fun); + visit(fun.getArgs()); + } + + public void visit(List expressions) { + for (DorisExpression expression : expressions) { + visit(expression); + } + } + + public void visit(DorisBetweenOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getMiddle()); + visit(op.getRight()); + } + + public void visit(DorisInOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(DorisOrderByTerm op) { + print(op); + visit(op.getExpr()); + } + + public void visit(DorisAlias op) { + print(op); + visit(op.getExpr()); + } + + public void visit(DorisPostfixText postFixText) { + print(postFixText); + visit(postFixText.getExpr()); + } + + public void visit(DorisConstant constant) { + print(constant); + } + + public void visit(DorisSelect select) { + print(select.getWhereClause()); + } + + public void visit(DorisJoin join) { + print(join.getOnCondition()); + } + + public String get() { + return sb.toString(); + } + + public static String asExpectedValues(DorisExpression expr) { + DorisExpectedValueVisitor v = new DorisExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + +} diff --git a/src/sqlancer/doris/visitor/DorisToStringVisitor.java b/src/sqlancer/doris/visitor/DorisToStringVisitor.java new file mode 100644 index 000000000..fcea9ab69 --- /dev/null +++ b/src/sqlancer/doris/visitor/DorisToStringVisitor.java @@ -0,0 +1,168 @@ +package sqlancer.doris.visitor; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.doris.ast.DorisCaseOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunctionOperation; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; + +public class DorisToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(DorisExpression expr) { + if (expr instanceof DorisConstant) { + visit((DorisConstant) expr); + } else if (expr instanceof DorisSelect) { + visit((DorisSelect) expr); + } else if (expr instanceof DorisJoin) { + visit((DorisJoin) expr); + } else if (expr instanceof DorisCastOperation) { + visit((DorisCastOperation) expr); + } else if (expr instanceof DorisCaseOperation) { + visit((DorisCaseOperation) expr); + } else if (expr instanceof DorisFunctionOperation) { + visit((DorisFunctionOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DorisJoin join) { + sb.append(" "); + visit((DorisExpression) join.getLeftTable()); + sb.append(" "); + switch (join.getJoinType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } else { + sb.append("CROSS "); + } + sb.append("JOIN "); + break; + case LEFT: + sb.append("LEFT "); + if (Randomly.getBoolean()) { + sb.append(" OUTER "); + } + sb.append("JOIN "); + break; + case RIGHT: + sb.append("RIGHT "); + if (Randomly.getBoolean()) { + sb.append(" OUTER "); + } + sb.append("JOIN "); + break; + case STRAIGHT: + sb.append("STRAIGHT_JOIN "); + break; + default: + throw new AssertionError(); + } + visit((DorisExpression) join.getRightTable()); + sb.append(" "); + if (join.getOnCondition() != null) { + sb.append("ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DorisConstant constant) { + sb.append(constant.toString()); + } + + private void visit(DorisCastOperation castExpr) { + sb.append("CAST("); + visit(castExpr.getExpr()); + sb.append(" AS "); + sb.append(castExpr.getType().toString()); + sb.append(") "); + } + + private void visit(DorisFunctionOperation func) { + sb.append(func.getFunction().getFunctionName()); + sb.append("("); + + if (func.getArgs() != null) { + for (int i = 0; i < func.getArgs().size(); i++) { + visit(func.getArgs().get(i)); + if (i != func.getArgs().size() - 1) { + sb.append(","); + } + } + } + sb.append(") "); + } + + private void visit(DorisCaseOperation cases) { + sb.append("CASE "); + visit(cases.getExpr()); + sb.append(" "); + for (int i = 0; i < cases.getConditions().size(); i++) { + DorisExpression predicate = cases.getConditions().get(i); + DorisExpression then = cases.getThenClauses().get(i); + sb.append(" WHEN "); + visit(predicate); + sb.append(" THEN "); + visit(then); + sb.append(" "); + } + if (cases.getElseClause() != null) { + sb.append("ELSE "); + visit(cases.getElseClause()); + sb.append(" "); + } + sb.append("END "); + } + + private void visit(DorisSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(DorisExpression expr) { + DorisToStringVisitor visitor = new DorisToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/duckdb/DuckDBBugs.java b/src/sqlancer/duckdb/DuckDBBugs.java new file mode 100644 index 000000000..c91661cb5 --- /dev/null +++ b/src/sqlancer/duckdb/DuckDBBugs.java @@ -0,0 +1,8 @@ +package sqlancer.duckdb; + +public final class DuckDBBugs { + + private DuckDBBugs() { + } + +} diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 61c940f2f..394de69d4 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -1,19 +1,27 @@ package sqlancer.duckdb; -import java.util.Set; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import sqlancer.common.query.ExpectedErrors; public final class DuckDBErrors { private DuckDBErrors() { } - public static void addExpressionErrors(Set errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("with non-constant precision is not supported"); + errors.add("Like pattern must not end with escape character"); errors.add("Could not convert string"); errors.add("ORDER term out of range - should be between "); errors.add("You might need to add explicit type casts."); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("field value out of range"); - errors.add("Not implemented: Unimplemented type for cast"); + errors.add("Unimplemented type for cast"); errors.add("Escape string must be empty or one character."); errors.add("Type mismatch when combining rows"); // BETWEEN @@ -21,77 +29,126 @@ public static void addExpressionErrors(Set errors) { errors.add("invalid UTF-8"); // TODO errors.add("String value is not valid UTF8"); - errors.add("Conversion: Invalid TypeId "); // TODO + errors.add("Invalid TypeId "); // TODO errors.add("GROUP BY clause cannot contain aggregates!"); // investigate - addRegexErrors(errors); + errors.addAll(getRegexErrors()); - addFunctionErrors(errors); + errors.addAll(getFunctionErrors()); errors.add("Overflow in multiplication"); errors.add("Out of Range"); - errors.add("Conversion: Date out of range"); + errors.add("Date out of range"); // collate errors.add("Cannot combine types with different collation!"); errors.add("collations are only supported for type varchar"); + errors.add("COLLATE can only be applied to varchar columns"); - // // https://github.com/cwida/duckdb/issues/532 - errors.add("Not implemented type: DATE"); - errors.add("Not implemented type: TIMESTAMP"); - - errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a - // table supports rowids + errors.add("Like pattern must not end with escape character!"); // LIKE errors.add("does not have a column named"); // TODO: this only happens for views whose underlying table has a // removed column errors.add("Contents of view were altered: types don't match!"); + errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + errors.add("ORDER BY non-integer literal has no effect"); + + // timestamp + errors.add("Cannot subtract infinite timestamps"); + errors.add("Timestamp difference is out of bounds"); + + return errors; + } + + public static List getExpressionErrorsRegex() { + ArrayList errors = new ArrayList<>(); + + errors.add(Pattern.compile("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause")); + errors.add(Pattern.compile("Binder Error: Cannot mix values of type .* and .* in CASE expression")); + errors.add(Pattern.compile("Cannot mix values of type .* and .* in COALESCE operator")); + errors.add(Pattern.compile("Cannot compare values of type .* and type .*")); + + return errors; } - private static void addRegexErrors(Set errors) { - errors.add("missing ]"); - errors.add("missing )"); - errors.add("invalid escape sequence"); - errors.add("no argument for repetition operator: "); - errors.add("bad repetition operator"); - errors.add("trailing \\"); - errors.add("invalid perl operator"); - errors.add("invalid character class range"); - errors.add("width is not integer"); + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + errors.addAllRegexes(getExpressionErrorsRegex()); + } + + private static List getRegexErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Invalid Input Error:"); + return errors; } - private static void addFunctionErrors(Set errors) { - errors.add("SUBSTRING cannot handle negative offsets"); + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("SUBSTRING cannot handle negative lengths"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF errors.add("argument index out of range"); // PRINTF errors.add("invalid format string"); // PRINTF errors.add("number is too big"); // PRINTF + errors.add("Like pattern must not end with escape character!"); // LIKE errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part errors.add("extract specifier"); // date_part + errors.add("not recognized"); // date_part + errors.add("not supported"); // date_part + errors.add("Failed to cast"); + errors.add("Conversion Error"); + errors.add("Could not cast value"); errors.add("Insufficient padding in RPAD"); // RPAD errors.add("Could not choose a best candidate function for the function call"); // monthname + errors.add("expected a numeric precision field"); // ROUND + errors.add("with non-constant precision is not supported"); // ROUND + + return errors; } - public static void addInsertErrors(Set errors) { + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getRegexErrors()); + errors.addAll(getFunctionErrors()); + errors.add("NOT NULL constraint failed"); - errors.add("PRIMARY KEY or UNIQUE constraint violated"); - errors.add("duplicate key value violates primary key or unique constraint"); + errors.add("PRIMARY KEY or UNIQUE constraint violation"); + errors.add("Duplicate key"); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("Could not convert string"); - errors.add("timestamp field value out of range"); - errors.add("Not implemented: Unimplemented type for cast"); // TODO: report? - errors.add("date/time field value out of range"); + errors.add("Unimplemented type for cast"); + errors.add("field value out of range"); errors.add("CHECK constraint failed"); errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if // the table has a primary key + errors.add("Could not cast value"); + errors.add("create unique index, table contains duplicate data"); + errors.add("Failed to cast"); + + return errors; } - public static void addGroupByErrors(Set errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("must appear in the GROUP BY clause or must be part of an aggregate function"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY term out of range"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); } } diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index bd33a81d4..00f85eece 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -1,25 +1,15 @@ package sqlancer.duckdb; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; -import sqlancer.TestOracle; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.test.DuckDBNoRECOracle; -import sqlancer.duckdb.test.DuckDBQueryPartitioningAggregateTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningDistinctTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningGroupByTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningHavingTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningWhereTester; +import sqlancer.DBMSSpecificOptions; -@Parameters -public class DuckDBOptions { +@Parameters(commandDescription = "DuckDB") +public class DuckDBOptions implements DBMSSpecificOptions { @Parameter(names = "--test-collate", arity = 1) public boolean testCollate = true; @@ -88,64 +78,11 @@ public class DuckDBOptions { public int maxNumUpdates = 5; @Parameter(names = "--oracle") - public List oracle = Arrays.asList(DuckDBOracle.QUERY_PARTITIONING); - - public enum DuckDBOracle { - NOREC { - - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); - oracles.add(new DuckDBQueryPartitioningWhereTester(globalState)); - oracles.add(new DuckDBQueryPartitioningHavingTester(globalState)); - oracles.add(new DuckDBQueryPartitioningAggregateTester(globalState)); - oracles.add(new DuckDBQueryPartitioningDistinctTester(globalState)); - oracles.add(new DuckDBQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles); - } - }; - - public abstract TestOracle create(DuckDBGlobalState globalState) throws SQLException; + public List oracles = Arrays.asList(DuckDBOracleFactory.QUERY_PARTITIONING); + @Override + public List getTestOracleFactory() { + return oracles; } } diff --git a/src/sqlancer/duckdb/DuckDBOracleFactory.java b/src/sqlancer/duckdb/DuckDBOracleFactory.java new file mode 100644 index 000000000..8fd0f96af --- /dev/null +++ b/src/sqlancer/duckdb/DuckDBOracleFactory.java @@ -0,0 +1,86 @@ +package sqlancer.duckdb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator; +import sqlancer.duckdb.test.DuckDBQueryPartitioningAggregateTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningDistinctTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningGroupByTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningHavingTester; + +public enum DuckDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .withRegex(DuckDBErrors.getExpressionErrorsRegex()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .with(DuckDBErrors.getGroupByErrors()).withRegex(DuckDBErrors.getExpressionErrorsRegex()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index ab6399b7e..59b26eac7 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -1,26 +1,25 @@ package sqlancer.duckdb; -import java.io.IOException; +import java.io.File; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.HashSet; -import java.util.Set; -import java.util.stream.Collectors; +import java.sql.Statement; + +import com.google.auto.service.AutoService; import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; -import sqlancer.GlobalState; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; -import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; +import sqlancer.MainOptions; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.gen.DuckDBDeleteGenerator; import sqlancer.duckdb.gen.DuckDBIndexGenerator; @@ -30,7 +29,8 @@ import sqlancer.duckdb.gen.DuckDBUpdateGenerator; import sqlancer.duckdb.gen.DuckDBViewGenerator; -public class DuckDBProvider extends ProviderAdapter { +@AutoService(DatabaseProvider.class) +public class DuckDBProvider extends SQLProviderAdapter { public DuckDBProvider() { super(DuckDBGlobalState.class, DuckDBOptions.class); @@ -40,30 +40,30 @@ public enum Action implements AbstractAction { INSERT(DuckDBInsertGenerator::getQuery), // CREATE_INDEX(DuckDBIndexGenerator::getQuery), // - VACUUM((g) -> new QueryAdapter("VACUUM;")), // - ANALYZE((g) -> new QueryAdapter("ANALYZE;")), // + VACUUM((g) -> new SQLQueryAdapter("VACUUM;")), // + ANALYZE((g) -> new SQLQueryAdapter("ANALYZE;")), // DELETE(DuckDBDeleteGenerator::generate), // UPDATE(DuckDBUpdateGenerator::getQuery), // CREATE_VIEW(DuckDBViewGenerator::generate), // EXPLAIN((g) -> { - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); DuckDBErrors.addExpressionErrors(errors); DuckDBErrors.addGroupByErrors(errors); - return new QueryAdapter( + return new SQLQueryAdapter( "EXPLAIN " + DuckDBToStringVisitor .asString(DuckDBRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), errors); }); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(DuckDBGlobalState state) throws SQLException { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(DuckDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } @@ -73,108 +73,87 @@ private static int mapActions(DuckDBGlobalState globalState, Action a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case CREATE_INDEX: - if (!globalState.getDmbsSpecificOptions().testIndexes) { + if (!globalState.getDbmsSpecificOptions().testIndexes) { return 0; } // fall through case UPDATE: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumUpdates + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); case VACUUM: // seems to be ignored case ANALYZE: // seems to be ignored case EXPLAIN: return r.getInteger(0, 2); case DELETE: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumDeletes + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); case CREATE_VIEW: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumViews + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } } - public static class DuckDBGlobalState extends GlobalState { - - private DuckDBSchema schema; + public static class DuckDBGlobalState extends SQLGlobalState { - public void setSchema(DuckDBSchema schema) { - this.schema = schema; - } - - public DuckDBSchema getSchema() { - return schema; + @Override + protected DuckDBSchema readSchema() throws SQLException { + return DuckDBSchema.fromConnection(getConnection(), getDatabaseName()); } } @Override - public void generateAndTestDatabase(DuckDBGlobalState globalState) throws SQLException { - StateLogger logger = globalState.getLogger(); - QueryManager manager = globalState.getManager(); - globalState.setSchema(DuckDBSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName())); - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(globalState.getState()); - } + public void generateDatabase(DuckDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { - boolean success = false; + boolean success; do { - Query qt = new DuckDBTableGenerator().getQuery(globalState); - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(qt.getQueryString()); - } - success = manager.execute(qt); - globalState.setSchema( - DuckDBSchema.fromConnection(globalState.getConnection(), globalState.getDatabaseName())); + SQLQueryAdapter qt = new DuckDBTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); } while (!success); } - if (globalState.getSchema().getDatabaseTables().size() == 0) { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); // TODO } StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), DuckDBProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - globalState.setSchema(DuckDBSchema.fromConnection(globalState.getConnection(), - globalState.getDatabaseName())); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } }); se.executeStatements(); - manager.incrementCreateDatabase(); - - TestOracle oracle = new CompositeTestOracle(globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (SQLException e1) { - throw new AssertionError(e1); - } - }).collect(Collectors.toList())); - - for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { + } - } - } + public void tryDeleteFile(String fname) { try { - if (globalState.getOptions().logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); + File f = new File(fname); + f.delete(); + } catch (Exception e) { } - globalState.getConnection().close(); + } + + public void tryDeleteDatabase(String dbpath) { + if (dbpath.equals("") || dbpath.equals(":memory:")) { + return; + } + tryDeleteFile(dbpath); + tryDeleteFile(dbpath + ".wal"); } @Override - public Connection createDatabase(DuckDBGlobalState globalState) throws SQLException { - String url = "jdbc:duckdb:"; - return DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); + public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLException { + String databaseFile = System.getProperty("duckdb.database.file", ""); + String url = "jdbc:duckdb:" + databaseFile; + tryDeleteDatabase(databaseFile); + + MainOptions options = globalState.getOptions(); + if (!(options.isDefaultUsername() && options.isDefaultPassword())) { + throw new AssertionError("DuckDB doesn't support credentials (username/password)"); + } + + Connection conn = DriverManager.getConnection(url); + Statement stmt = conn.createStatement(); + stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); + stmt.close(); + return new SQLConnection(conn); } @Override diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 5daec973b..857b1e008 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -1,29 +1,35 @@ package sqlancer.duckdb; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; -public class DuckDBSchema extends AbstractSchema { +public class DuckDBSchema extends AbstractSchema { public enum DuckDBDataType { - INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP; + INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP, NULL; - public static DuckDBDataType getRandom() { - return Randomly.fromOptions(values()); + public static DuckDBDataType getRandomWithoutNull() { + DuckDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DuckDBDataType.NULL); + return dt; } } @@ -34,11 +40,6 @@ public static class DuckDBCompositeDataType { private final int size; - public DuckDBCompositeDataType(DuckDBDataType dataType) { - this.dataType = dataType; - this.size = -1; - } - public DuckDBCompositeDataType(DuckDBDataType dataType, int size) { this.dataType = dataType; this.size = size; @@ -55,8 +56,8 @@ public int getSize() { return size; } - public static DuckDBCompositeDataType getRandom() { - DuckDBDataType type = DuckDBDataType.getRandom(); + public static DuckDBCompositeDataType getRandomWithoutNull() { + DuckDBDataType type = DuckDBDataType.getRandomWithoutNull(); int size = -1; switch (type) { case INT: @@ -78,10 +79,6 @@ public static DuckDBCompositeDataType getRandom() { return new DuckDBCompositeDataType(type, size); } - public static DuckDBCompositeDataType getInt(int size) { - return new DuckDBCompositeDataType(DuckDBDataType.INT, size); - } - @Override public String toString() { switch (getPrimitiveDataType()) { @@ -103,7 +100,7 @@ public String toString() { case FLOAT: switch (size) { case 8: - return Randomly.fromOptions("DOUBLE", "NUMERIC"); + return Randomly.fromOptions("DOUBLE"); case 4: return Randomly.fromOptions("REAL", "FLOAT4"); default: @@ -115,6 +112,8 @@ public String toString() { return Randomly.fromOptions("TIMESTAMP", "DATETIME"); case DATE: return Randomly.fromOptions("DATE"); + case NULL: + return Randomly.fromOptions("NULL"); default: throw new AssertionError(getPrimitiveDataType()); } @@ -133,6 +132,7 @@ public DuckDBColumn(String name, DuckDBCompositeDataType columnType, boolean isP this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -162,6 +162,9 @@ public DuckDBTables getRandomTableNonEmptyTables() { private static DuckDBCompositeDataType getColumnType(String typeString) { DuckDBDataType primitiveType; int size = -1; + if (typeString.startsWith("DECIMAL")) { // Ugly hack + return new DuckDBCompositeDataType(DuckDBDataType.FLOAT, 8); + } switch (typeString) { case "INTEGER": primitiveType = DuckDBDataType.INT; @@ -172,6 +175,7 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { size = 2; break; case "BIGINT": + case "HUGEINT": // TODO: 16-bit int primitiveType = DuckDBDataType.INT; size = 8; break; @@ -199,31 +203,38 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { case "TIMESTAMP": primitiveType = DuckDBDataType.TIMESTAMP; break; + case "\"NULL\"": + primitiveType = DuckDBDataType.NULL; + break; + case "INTERVAL": + throw new IgnoreMeException(); + // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP + // '1969-12-29 06:50:27')) default: throw new AssertionError(typeString); } return new DuckDBCompositeDataType(primitiveType, size); } - public static class DuckDBTable extends AbstractTable { - - public DuckDBTable(String tableName, List columns, boolean isView) { - super(tableName, columns, Collections.emptyList(), isView); - } + public static class DuckDBTable extends AbstractRelationalTable { - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); + public DuckDBTable(String tableName, List columns, List indexes, boolean isView) { + super(tableName, columns, indexes, isView); } } - public static DuckDBSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static DuckDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); - DuckDBTable t = new DuckDBTable(tableName, databaseColumns, isView); + boolean isView = matchesViewName(tableName); + List indexes = getIndexes(con, tableName); + DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { c.setTable(t); } @@ -233,10 +244,25 @@ public static DuckDBSchema fromConnection(Connection con, String databaseName) t return new DuckDBSchema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getIndexes(SQLConnection con, String tableName) throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT index_name FROM duckdb_indexes() WHERE database_name = current_database() AND table_name = '%s';", + tableName))) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + indexes.add(TableIndex.create(indexName)); + } + } + } + return indexes; + } + + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master()")) { + try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master WHERE type='table' or type='view'")) { while (rs.next()) { tableNames.add(rs.getString("name")); } @@ -245,7 +271,7 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format("SELECT * FROM pragma_table_info('%s');", tableName))) { @@ -259,7 +285,7 @@ private static List getTableColumns(Connection con, String tableNa } } } - if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { + if (columns.stream().noneMatch(c -> c.isPrimaryKey()) && !AbstractSchema.matchesViewName(tableName)) { // https://github.com/cwida/duckdb/issues/589 // https://github.com/cwida/duckdb/issues/588 // TODO: implement an option to enable/disable rowids diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index f35e3445e..b1a8eb703 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -1,7 +1,7 @@ package sqlancer.duckdb; -import sqlancer.ast.newast.NewToStringVisitor; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.ast.DuckDBConstant; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; @@ -10,7 +10,7 @@ public class DuckDBToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DuckDBExpression expr) { if (expr instanceof DuckDBConstant) { visit((DuckDBConstant) expr); } else if (expr instanceof DuckDBSelect) { @@ -23,7 +23,7 @@ public void visitSpecific(Node expr) { } private void visit(DuckDBJoin join) { - visit(join.getLeftTable()); + visit((TableReferenceNode) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); @@ -31,7 +31,7 @@ private void visit(DuckDBJoin join) { sb.append(join.getOuterType()); } sb.append(" JOIN "); - visit(join.getRightTable()); + visit((TableReferenceNode) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); @@ -68,9 +68,9 @@ private void visit(DuckDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); @@ -82,7 +82,7 @@ private void visit(DuckDBSelect select) { } } - public static String asString(Node expr) { + public static String asString(DuckDBExpression expr) { DuckDBToStringVisitor visitor = new DuckDBToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/duckdb/ast/DuckDBAlias.java b/src/sqlancer/duckdb/ast/DuckDBAlias.java new file mode 100644 index 000000000..cc1a91921 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBAlias.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DuckDBAlias extends NewAliasNode implements DuckDBExpression { + public DuckDBAlias(DuckDBExpression expr, String string) { + super(expr, string); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java b/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java new file mode 100644 index 000000000..eabaa6162 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class DuckDBBetweenOperator extends NewBetweenOperatorNode implements DuckDBExpression { + public DuckDBBetweenOperator(DuckDBExpression left, DuckDBExpression middle, DuckDBExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java b/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java new file mode 100644 index 000000000..24492c145 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DuckDBBinaryOperator extends NewBinaryOperatorNode implements DuckDBExpression { + public DuckDBBinaryOperator(DuckDBExpression left, DuckDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java b/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java new file mode 100644 index 000000000..ba1d9d96f --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java @@ -0,0 +1,12 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class DuckDBCaseOperator extends NewCaseOperatorNode implements DuckDBExpression { + public DuckDBCaseOperator(DuckDBExpression switchCondition, List conditions, + List expressions, DuckDBExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBColumnReference.java b/src/sqlancer/duckdb/ast/DuckDBColumnReference.java new file mode 100644 index 000000000..d24382777 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.duckdb.DuckDBSchema; + +public class DuckDBColumnReference extends ColumnReferenceNode + implements DuckDBExpression { + public DuckDBColumnReference(DuckDBSchema.DuckDBColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/duckdb/ast/DuckDBConstant.java b/src/sqlancer/duckdb/ast/DuckDBConstant.java index d5b65e50d..f4af6918a 100644 --- a/src/sqlancer/duckdb/ast/DuckDBConstant.java +++ b/src/sqlancer/duckdb/ast/DuckDBConstant.java @@ -3,9 +3,7 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.ast.newast.Node; - -public class DuckDBConstant implements Node { +public class DuckDBConstant implements DuckDBExpression { private DuckDBConstant() { } @@ -161,31 +159,31 @@ public String toString() { } - public static Node createStringConstant(String text) { + public static DuckDBExpression createStringConstant(String text) { return new DuckDBTextConstant(text); } - public static Node createFloatConstant(double val) { + public static DuckDBExpression createFloatConstant(double val) { return new DuckDBDoubleConstant(val); } - public static Node createIntConstant(long val) { + public static DuckDBExpression createIntConstant(long val) { return new DuckDBIntConstant(val); } - public static Node createNullConstant() { + public static DuckDBExpression createNullConstant() { return new DuckDBNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static DuckDBExpression createBooleanConstant(boolean val) { return new DuckDBBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static DuckDBExpression createDateConstant(long integer) { return new DuckDBDateConstant(integer); } - public static Node createTimestampConstant(long integer) { + public static DuckDBExpression createTimestampConstant(long integer) { return new DuckDBTimestampConstant(integer); } diff --git a/src/sqlancer/duckdb/ast/DuckDBExpression.java b/src/sqlancer/duckdb/ast/DuckDBExpression.java index 3ff66ce33..21d4e6eb5 100644 --- a/src/sqlancer/duckdb/ast/DuckDBExpression.java +++ b/src/sqlancer/duckdb/ast/DuckDBExpression.java @@ -1,5 +1,7 @@ package sqlancer.duckdb.ast; -public interface DuckDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +public interface DuckDBExpression extends Expression { } diff --git a/src/sqlancer/duckdb/ast/DuckDBFunction.java b/src/sqlancer/duckdb/ast/DuckDBFunction.java new file mode 100644 index 000000000..445583090 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBFunction.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DuckDBFunction extends NewFunctionNode implements DuckDBExpression { + public DuckDBFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBInOperator.java b/src/sqlancer/duckdb/ast/DuckDBInOperator.java new file mode 100644 index 000000000..619601749 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBInOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class DuckDBInOperator extends NewInOperatorNode implements DuckDBExpression { + public DuckDBInOperator(DuckDBExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBJoin.java b/src/sqlancer/duckdb/ast/DuckDBJoin.java index 42e0ea61f..f677e9efc 100644 --- a/src/sqlancer/duckdb/ast/DuckDBJoin.java +++ b/src/sqlancer/duckdb/ast/DuckDBJoin.java @@ -4,19 +4,18 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; -import sqlancer.ast.newast.TableReferenceNode; +import sqlancer.common.ast.newast.Join; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -public class DuckDBJoin implements Node { +public class DuckDBJoin implements DuckDBExpression, Join { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DuckDBTableReference leftTable; + private final DuckDBTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private DuckDBExpression onCondition; private OuterType outerType; public enum JoinType { @@ -35,20 +34,19 @@ public static OuterType getRandom() { } } - public DuckDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DuckDBJoin(DuckDBTableReference leftTable, DuckDBTableReference rightTable, JoinType joinType, + DuckDBExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public DuckDBTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public DuckDBTableReference getRightTable() { return rightTable; } @@ -56,7 +54,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DuckDBExpression getOnCondition() { return onCondition; } @@ -68,12 +66,11 @@ public OuterType getOuterType() { return outerType; } - public static List> getJoins( - List> tableList, DuckDBGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DuckDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DuckDBTableReference leftTable = tableList.remove(0); + DuckDBTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); DuckDBExpressionGenerator joinGen = new DuckDBExpressionGenerator(globalState).setColumns(columns); @@ -99,26 +96,30 @@ public static List> getJoins( return joinExpressions; } - public static DuckDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createRightOuterJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.RIGHT, predicate); } - public static DuckDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createLeftOuterJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.LEFT, predicate); } - public static DuckDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createInnerJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + public static DuckDBJoin createNaturalJoin(DuckDBTableReference left, DuckDBTableReference right, + OuterType naturalJoinType) { DuckDBJoin join = new DuckDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; } + @Override + public void setOnClause(DuckDBExpression onClause) { + this.onCondition = onClause; + } } diff --git a/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java b/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java new file mode 100644 index 000000000..8d7177655 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class DuckDBOrderingTerm extends NewOrderingTerm implements DuckDBExpression { + public DuckDBOrderingTerm(DuckDBExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBPostFixText.java b/src/sqlancer/duckdb/ast/DuckDBPostFixText.java new file mode 100644 index 000000000..a877a4c92 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBPostFixText.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DuckDBPostFixText extends NewPostfixTextNode implements DuckDBExpression { + public DuckDBPostFixText(DuckDBExpression expr, String string) { + super(expr, string); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBSelect.java b/src/sqlancer/duckdb/ast/DuckDBSelect.java index 5e92aa685..e18e57a4d 100644 --- a/src/sqlancer/duckdb/ast/DuckDBSelect.java +++ b/src/sqlancer/duckdb/ast/DuckDBSelect.java @@ -1,9 +1,16 @@ package sqlancer.duckdb.ast; -import sqlancer.ast.SelectBase; -import sqlancer.ast.newast.Node; +import java.util.List; +import java.util.stream.Collectors; -public class DuckDBSelect extends SelectBase> implements Node { +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.DuckDBToStringVisitor; + +public class DuckDBSelect extends SelectBase + implements Select, DuckDBExpression { private boolean isDistinct; @@ -15,4 +22,20 @@ public boolean isDistinct() { return isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DuckDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DuckDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DuckDBToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/duckdb/ast/DuckDBTableReference.java b/src/sqlancer/duckdb/ast/DuckDBTableReference.java new file mode 100644 index 000000000..0a8d795c8 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.duckdb.DuckDBSchema; + +public class DuckDBTableReference extends TableReferenceNode + implements DuckDBExpression { + public DuckDBTableReference(DuckDBSchema.DuckDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBTernary.java b/src/sqlancer/duckdb/ast/DuckDBTernary.java new file mode 100644 index 000000000..921f77b90 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBTernary.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewTernaryNode; + +public class DuckDBTernary extends NewTernaryNode implements DuckDBExpression { + public DuckDBTernary(DuckDBExpression left, DuckDBExpression middle, DuckDBExpression right, String leftString, + String rightString) { + super(left, middle, right, leftString, rightString); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java b/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java new file mode 100644 index 000000000..856c129ec --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class DuckDBUnaryPostfixOperator extends NewUnaryPostfixOperatorNode + implements DuckDBExpression { + public DuckDBUnaryPostfixOperator(DuckDBExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java b/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java new file mode 100644 index 000000000..95c8b4990 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class DuckDBUnaryPrefixOperator extends NewUnaryPrefixOperatorNode + implements DuckDBExpression { + public DuckDBUnaryPrefixOperator(DuckDBExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java index 01d65213f..710fca1e2 100644 --- a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java @@ -1,11 +1,8 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; @@ -21,8 +18,8 @@ enum Action { ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN } - public static Query getQuery(DuckDBGlobalState globalState) { - Set errors = new HashSet<>(); + public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); errors.add(" does not have a column with name \"rowid\""); errors.add("Table does not contain column rowid referenced in alter statement"); StringBuilder sb = new StringBuilder("ALTER TABLE "); @@ -37,13 +34,13 @@ public static Query getQuery(DuckDBGlobalState globalState) { String columnName = table.getFreeColumnName(); sb.append(columnName); sb.append(" "); - sb.append(DuckDBCompositeDataType.getRandom().toString()); + sb.append(DuckDBCompositeDataType.getRandomWithoutNull().toString()); break; case ALTER_COLUMN: sb.append("ALTER COLUMN "); sb.append(table.getRandomColumn().getName()); sb.append(" SET DATA TYPE "); - sb.append(DuckDBCompositeDataType.getRandom().toString()); + sb.append(DuckDBCompositeDataType.getRandomWithoutNull().toString()); if (Randomly.getBoolean()) { sb.append(" USING "); DuckDBErrors.addExpressionErrors(errors); @@ -67,7 +64,7 @@ public static Query getQuery(DuckDBGlobalState globalState) { default: throw new AssertionError(action); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 248bc40c2..9f4f4ed6a 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -1,33 +1,34 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBToStringVisitor; -public final class DuckDBDeleteGenerator { +public final class DuckDBDeleteGenerator extends AbstractDeleteGenerator { + + private final DuckDBGlobalState globalState; + + private DuckDBDeleteGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; + } - private DuckDBDeleteGenerator() { + public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { + return new DuckDBDeleteGenerator(globalState).getStatement(); } - public static Query generate(DuckDBGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - Set errors = new HashSet<>(); + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DuckDBToStringVisitor.asString( + appendWhereClause(DuckDBToStringVisitor.asString( new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } DuckDBErrors.addExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index f00685dde..278685a46 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -3,33 +3,44 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.NewBetweenOperatorNode; -import sqlancer.ast.newast.NewBinaryOperatorNode; -import sqlancer.ast.newast.NewCaseOperatorNode; -import sqlancer.ast.newast.NewFunctionNode; -import sqlancer.ast.newast.NewInOperatorNode; -import sqlancer.ast.newast.NewOrderingTerm; -import sqlancer.ast.newast.NewOrderingTerm.Ordering; -import sqlancer.ast.newast.NewTernaryNode; -import sqlancer.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBBetweenOperator; +import sqlancer.duckdb.ast.DuckDBBinaryOperator; +import sqlancer.duckdb.ast.DuckDBCaseOperator; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBConstant; import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.gen.UntypedExpressionGenerator; - -public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator, DuckDBColumn> { +import sqlancer.duckdb.ast.DuckDBFunction; +import sqlancer.duckdb.ast.DuckDBInOperator; +import sqlancer.duckdb.ast.DuckDBJoin; +import sqlancer.duckdb.ast.DuckDBOrderingTerm; +import sqlancer.duckdb.ast.DuckDBPostFixText; +import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; +import sqlancer.duckdb.ast.DuckDBTernary; + +public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator + implements NoRECGenerator, + TLPWhereGenerator { private final DuckDBGlobalState globalState; + private List tables; public DuckDBExpressionGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; @@ -41,80 +52,78 @@ private enum Expression { } @Override - protected Node generateExpression(int depth) { + protected DuckDBExpression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } if (allowAggregates && Randomly.getBoolean()) { DuckDBAggregateFunction aggregate = DuckDBAggregateFunction.getRandom(); allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); + return new DuckDBFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - if (!globalState.getDmbsSpecificOptions().testCollate) { + if (!globalState.getDbmsSpecificOptions().testCollate) { possibleOptions.remove(Expression.COLLATE); } - if (!globalState.getDmbsSpecificOptions().testFunctions) { + if (!globalState.getDbmsSpecificOptions().testFunctions) { possibleOptions.remove(Expression.FUNC); } - if (!globalState.getDmbsSpecificOptions().testCasts) { + if (!globalState.getDbmsSpecificOptions().testCasts) { possibleOptions.remove(Expression.CAST); } - if (!globalState.getDmbsSpecificOptions().testBetween) { + if (!globalState.getDbmsSpecificOptions().testBetween) { possibleOptions.remove(Expression.BETWEEN); } - if (!globalState.getDmbsSpecificOptions().testIn) { + if (!globalState.getDbmsSpecificOptions().testIn) { possibleOptions.remove(Expression.IN); } - if (!globalState.getDmbsSpecificOptions().testCase) { + if (!globalState.getDbmsSpecificOptions().testCase) { possibleOptions.remove(Expression.CASE); } - if (!globalState.getDmbsSpecificOptions().testBinaryComparisons) { + if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { possibleOptions.remove(Expression.BINARY_COMPARISON); } - if (!globalState.getDmbsSpecificOptions().testBinaryLogicals) { + if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { possibleOptions.remove(Expression.BINARY_LOGICAL); } Expression expr = Randomly.fromList(possibleOptions); switch (expr) { case COLLATE: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(generateExpression(depth + 1), DuckDBCollate.getRandom()); case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(generateExpression(depth + 1), DuckDBUnaryPrefixOperator.getRandom()); case UNARY_POSTFIX: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(generateExpression(depth + 1), DuckDBUnaryPostfixOperator.getRandom()); case BINARY_COMPARISON: Operator op = DuckDBBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_LOGICAL: op = DuckDBBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), DuckDBBinaryArithmeticOperator.getRandom()); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), + DuckDBBinaryArithmeticOperator.getRandom()); case CAST: - return new DuckDBCastOperation(generateExpression(depth + 1), DuckDBCompositeDataType.getRandom()); + return new DuckDBCastOperation(generateExpression(depth + 1), + DuckDBCompositeDataType.getRandomWithoutNull()); case FUNC: DBFunction func = DBFunction.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + return new DuckDBFunction<>(generateExpressions(func.getNrArgs()), func); case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + return new DuckDBBetweenOperator(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); case IN: - return new NewInOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + return new DuckDBInOperator(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, nr), generateExpressions(depth + 1, nr), - generateExpression(depth + 1)); + return new DuckDBCaseOperator(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case LIKE_ESCAPE: - return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), + return new DuckDBTernary(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), "LIKE", "ESCAPE"); default: throw new AssertionError(); @@ -122,45 +131,45 @@ protected Node generateExpression(int depth) { } @Override - protected Node generateColumn() { + protected DuckDBExpression generateColumn() { DuckDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode(column); + return new DuckDBColumnReference(column); } @Override - public Node generateConstant() { + public DuckDBExpression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return DuckDBConstant.createNullConstant(); } - DuckDBDataType type = DuckDBDataType.getRandom(); + DuckDBDataType type = DuckDBDataType.getRandomWithoutNull(); switch (type) { case INT: - if (!globalState.getDmbsSpecificOptions().testIntConstants) { + if (!globalState.getDbmsSpecificOptions().testIntConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createIntConstant(globalState.getRandomly().getInteger()); case DATE: - if (!globalState.getDmbsSpecificOptions().testDateConstants) { + if (!globalState.getDbmsSpecificOptions().testDateConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createDateConstant(globalState.getRandomly().getInteger()); case TIMESTAMP: - if (!globalState.getDmbsSpecificOptions().testTimestampConstants) { + if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); case VARCHAR: - if (!globalState.getDmbsSpecificOptions().testStringConstants) { + if (!globalState.getDbmsSpecificOptions().testStringConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createStringConstant(globalState.getRandomly().getString()); case BOOLEAN: - if (!globalState.getDmbsSpecificOptions().testBooleanConstants) { + if (!globalState.getDbmsSpecificOptions().testBooleanConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: - if (!globalState.getDmbsSpecificOptions().testFloatConstants) { + if (!globalState.getDbmsSpecificOptions().testFloatConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); @@ -170,21 +179,22 @@ public Node generateConstant() { } @Override - public List> generateOrderBys() { - List> expr = super.generateOrderBys(); - List> newExpr = new ArrayList<>(expr.size()); - for (Node curExpr : expr) { + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (DuckDBExpression curExpr : expr) { if (Randomly.getBoolean()) { - curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + curExpr = new DuckDBOrderingTerm(curExpr, Ordering.getRandom()); } newExpr.add(curExpr); } return newExpr; }; - public static class DuckDBCastOperation extends NewUnaryPostfixOperatorNode { + public static class DuckDBCastOperation extends NewUnaryPostfixOperatorNode + implements DuckDBExpression { - public DuckDBCastOperation(Node expr, DuckDBCompositeDataType type) { + public DuckDBCastOperation(DuckDBExpression expr, DuckDBCompositeDataType type) { super(expr, new Operator() { @Override @@ -243,6 +253,7 @@ public enum DBFunction { DEGREES(1), // RADIANS(1), // MOD(2), // + XOR(2), // // string functions LENGTH(1), // LOWER(1), // @@ -379,8 +390,7 @@ public static Operator getRandom() { } public enum DuckDBBinaryArithmeticOperator implements Operator { - CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), XOR("#"), LSHIFT("<<"), - RSHIFT(">>"); + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); private String textRepr; @@ -400,7 +410,6 @@ public String getTextRepresentation() { } public enum DuckDBBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); @@ -422,15 +431,96 @@ public String getTextRepresentation() { } - public NewFunctionNode generateArgsForAggregate( - DuckDBAggregateFunction aggregateFunction) { - return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + public DuckDBFunction generateArgsForAggregate(DuckDBAggregateFunction aggregateFunction) { + return new DuckDBFunction<>(generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); } - public Node generateAggregate() { + public DuckDBExpression generateAggregate() { DuckDBAggregateFunction aggrFunc = DuckDBAggregateFunction.getRandom(); return generateArgsForAggregate(aggrFunc); } + @Override + public DuckDBExpression negatePredicate(DuckDBExpression predicate) { + return new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(predicate, DuckDBUnaryPrefixOperator.NOT); + } + + @Override + public DuckDBExpression isNull(DuckDBExpression expr) { + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(expr, DuckDBUnaryPostfixOperator.IS_NULL); + } + + @Override + public DuckDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DuckDBExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public DuckDBSelect generateSelect() { + return new DuckDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DuckDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DuckDBSelect select, DuckDBExpression whereCondition, + boolean shouldUseAggregate) { + List allColumns = columns.stream().map((c) -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); + if (shouldUseAggregate) { + DuckDBFunction aggr = new DuckDBFunction<>( + Arrays.asList(new DuckDBColumnReference( + new DuckDBColumn("*", new DuckDBCompositeDataType(DuckDBDataType.INT, 0), false, false))), + DuckDBAggregateFunction.COUNT); + select.setFetchColumns(Arrays.asList(aggr)); + } else { + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DuckDBSelect select, DuckDBExpression whereCondition) { + DuckDBExpression asText = new DuckDBPostFixText(new DuckDBCastOperation( + new DuckDBPostFixText(whereCondition, + " IS NOT NULL AND " + DuckDBToStringVisitor.asString(whereCondition)), + new DuckDBCompositeDataType(DuckDBDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (Randomly.getBoolean()) { + return List.of(new DuckDBColumnReference(new DuckDBColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); + } } diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 44bcae79a..bd42b64bd 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -1,34 +1,35 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.DuckDBToStringVisitor; -import sqlancer.duckdb.ast.DuckDBExpression; -public final class DuckDBIndexGenerator { +public class DuckDBIndexGenerator extends AbstractIndexGenerator { - private DuckDBIndexGenerator() { + private final DuckDBGlobalState globalState; + + public DuckDBIndexGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + } + + public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { + return new DuckDBIndexGenerator(globalState).getStatement(); } - public static Query getQuery(DuckDBGlobalState globalState) { - Set errors = new HashSet<>(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { - errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { + errors.add("Data contains duplicates on indexed column(s)"); } - sb.append("INDEX "); - sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + appendCreateIndex(unique); + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); @@ -45,17 +46,9 @@ public static Query getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - Node expr = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()) - .generateExpression(); - sb.append(DuckDBToStringVisitor.asString(expr)); - } - errors.add("already exists!"); - if (globalState.getDmbsSpecificOptions().testRowid) { - errors.add("Cannot create an index on the rowid!"); + if (globalState.getDbmsSpecificOptions().testRowid) { + errors.add("cannot create an index on the rowid"); } - return new QueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 99369e9bf..e8b122a51 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -1,49 +1,38 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBToStringVisitor; -import sqlancer.gen.AbstractInsertGenerator; public class DuckDBInsertGenerator extends AbstractInsertGenerator { private final DuckDBGlobalState globalState; - private final Set errors = new HashSet<>(); public DuckDBInsertGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; } - public static Query getQuery(DuckDBGlobalState globalState) { - return new DuckDBInsertGenerator(globalState).generate(); + public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { + return new DuckDBInsertGenerator(globalState).getStatement(); } - private Query generate() { - sb.append("INSERT INTO "); + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); + buildInsertInto(table.getName(), columns); DuckDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); } @Override - protected void insertValue(DuckDBColumn tiDBColumn) { + protected void insertValue(DuckDBColumn columnDuckDB) { // TODO: select a more meaningful value if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("DEFAULT"); diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index f48c4474d..d88d4f0b8 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -5,8 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; -import sqlancer.ast.newast.TableReferenceNode; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; @@ -14,6 +12,7 @@ import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; public final class DuckDBRandomQuerySynthesizer { @@ -28,10 +27,10 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC // TODO: distinct // select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen.generateExpression(); + DuckDBExpression expression = gen.generateExpression(); columns.add(expression); // } else { // columns.add(gen()); @@ -39,16 +38,16 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression()); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index a048f6646..c8ac6f28d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -1,15 +1,13 @@ package sqlancer.duckdb.gen; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; +import sqlancer.common.gen.AbstractTableGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; @@ -17,53 +15,30 @@ import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.gen.UntypedExpressionGenerator; -public class DuckDBTableGenerator { +public class DuckDBTableGenerator extends AbstractTableGenerator { - public Query getQuery(DuckDBGlobalState globalState) { - Set errors = new HashSet<>(); - StringBuilder sb = new StringBuilder(); + private DuckDBGlobalState globalState; + private UntypedExpressionGenerator gen; + + public DuckDBTableGenerator() { + this.canAffectSchema = true; + } + + public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName); List columns = getNewColumns(); - UntypedExpressionGenerator, DuckDBColumn> gen = new DuckDBExpressionGenerator( - globalState).setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - if (globalState.getDmbsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() - && columns.get(i).getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { - sb.append(" COLLATE "); - sb.append(getRandomCollate()); - } - if (globalState.getDmbsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" UNIQUE"); - } - if (globalState.getDmbsSpecificOptions().testNotNullConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" NOT NULL"); - } - if (globalState.getDmbsSpecificOptions().testCheckConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" CHECK("); - sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); - DuckDBErrors.addExpressionErrors(errors); - sb.append(")"); - } - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().testDefaultValues) { - sb.append(" DEFAULT("); - sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); - sb.append(")"); - } - } - if (globalState.getDmbsSpecificOptions().testIndexes && Randomly.getBoolean()) { + gen = new DuckDBExpressionGenerator(globalState).setColumns(columns); + sb.append("("); + appendColumnDefinitionList(columns); + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); sb.append(", PRIMARY KEY("); @@ -71,7 +46,37 @@ public Query getQuery(DuckDBGlobalState globalState) { sb.append(")"); } sb.append(")"); - return new QueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void appendColumnDefinition(DuckDBColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + && column.getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { + sb.append(" COLLATE "); + sb.append(getRandomCollate()); + } + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" UNIQUE"); + } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } + if (globalState.getDbmsSpecificOptions().testCheckConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" CHECK("); + sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); + DuckDBErrors.addExpressionErrors(errors); + sb.append(")"); + } + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + } } public static String getRandomCollate() { @@ -82,7 +87,7 @@ private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); - DuckDBCompositeDataType columnType = DuckDBCompositeDataType.getRandom(); + DuckDBCompositeDataType columnType = DuckDBCompositeDataType.getRandomWithoutNull(); columns.add(new DuckDBColumn(columnName, columnType, false, false)); } return columns; diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 5c0b3a27e..8e2ddd047 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -1,13 +1,10 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.ast.newast.Node; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; @@ -15,36 +12,41 @@ import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public final class DuckDBUpdateGenerator { +public final class DuckDBUpdateGenerator extends AbstractUpdateGenerator { - private DuckDBUpdateGenerator() { + private final DuckDBGlobalState globalState; + private DuckDBExpressionGenerator gen; + + private DuckDBUpdateGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { + return new DuckDBUpdateGenerator(globalState).getStatement(); } - public static Query getQuery(DuckDBGlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - Set errors = new HashSet<>(); + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); + gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - if (Randomly.getBooleanWithSmallProbability()) { - expr = gen.generateExpression(); - DuckDBErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(); - } - sb.append(DuckDBToStringVisitor.asString(expr)); - } + updateColumns(columns); DuckDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); + } + + @Override + protected void updateValue(DuckDBColumn column) { + DuckDBExpression expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + DuckDBErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(DuckDBToStringVisitor.asString(expr)); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java index 194fcc6b3..c56f23b1c 100644 --- a/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBViewGenerator.java @@ -1,11 +1,8 @@ package sqlancer.duckdb.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBToStringVisitor; @@ -15,7 +12,7 @@ public final class DuckDBViewGenerator { private DuckDBViewGenerator() { } - public static Query generate(DuckDBGlobalState globalState) { + public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); @@ -25,14 +22,15 @@ public static Query generate(DuckDBGlobalState globalState) { if (i != 0) { sb.append(", "); } - sb.append("c" + i); + sb.append("c"); + sb.append(i); } sb.append(") AS "); sb.append(DuckDBToStringVisitor.asString(DuckDBRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); - Set errors = new HashSet<>(); + ExpectedErrors errors = new ExpectedErrors(); DuckDBErrors.addExpressionErrors(errors); DuckDBErrors.addGroupByErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java deleted file mode 100644 index 895b65ae5..000000000 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ /dev/null @@ -1,152 +0,0 @@ -package sqlancer.duckdb.test; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.Randomly; -import sqlancer.TestOracle; -import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.NewPostfixTextNode; -import sqlancer.ast.newast.Node; -import sqlancer.ast.newast.TableReferenceNode; -import sqlancer.duckdb.DuckDBErrors; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBSchema; -import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; -import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; -import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; -import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.DuckDBSchema.DuckDBTables; -import sqlancer.duckdb.DuckDBToStringVisitor; -import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.duckdb.ast.DuckDBJoin; -import sqlancer.duckdb.ast.DuckDBSelect; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBCastOperation; - -public class DuckDBNoRECOracle implements TestOracle { - - private final DuckDBSchema s; - private final Connection con; - private String firstQueryString; - private String secondQueryString; - private final StateLogger logger; - private final MainOptions options; - private final Set errors = new HashSet<>(); - private final DuckDBGlobalState globalState; - - public DuckDBNoRECOracle(DuckDBGlobalState globalState) { - this.s = globalState.getSchema(); - this.con = globalState.getConnection(); - this.logger = globalState.getLogger(); - this.options = globalState.getOptions(); - this.globalState = globalState; - DuckDBErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - // DuckDBCommon.addCommonExpressionErrors(errors); - // DuckDBCommon.addCommonFetchErrors(errors); - DuckDBTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, globalState); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); - int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - firstQueryString + "; -- " + firstCount + "\n" + secondQueryString + " -- " + secondCount); - } - } - - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { - DuckDBSelect select = new DuckDBSelect(); - // select.setGroupByClause(groupBys); - // DuckDBExpression isTrue = DuckDBPostfixOperation.create(randomWhereCondition, - // PostfixOperator.IS_TRUE); - Node asText = new NewPostfixTextNode<>(new DuckDBCastOperation( - new NewPostfixTextNode(randomWhereCondition, - " IS NOT NULL AND " + DuckDBToStringVisitor.asString(randomWhereCondition)), - new DuckDBCompositeDataType(DuckDBDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(tableList); - // select.setSelectType(SelectType.ALL); - select.setJoinList(joins); - int secondCount = 0; - secondQueryString = "SELECT SUM(count) FROM (" + DuckDBToStringVisitor.asString(select) + ") as res"; - errors.add("canceling statement due to statement timeout"); - Query q = new QueryAdapter(secondQueryString, errors); - ResultSet rs; - try { - rs = q.executeAndGetLogged(globalState); - } catch (Exception e) { - throw new AssertionError(secondQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getFirstQueryCount(Connection con, List> tableList, List columns, - Node randomWhereCondition, List> joins) throws SQLException { - DuckDBSelect select = new DuckDBSelect(); - // select.setGroupByClause(groupBys); - // DuckDBAggregate aggr = new DuckDBAggregate( - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - // DuckDBAggregateFunction.COUNT); - // select.setFetchColumns(Arrays.asList(aggr)); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( - new DuckDBExpressionGenerator(globalState).setColumns(columns).generateOrderBys()); - } - // select.setSelectType(SelectType.ALL); - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - firstQueryString = DuckDBToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(firstQueryString); - } - try (ResultSet rs = stat.executeQuery(firstQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; - } - -} diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index e46cff858..e82f97db6 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -1,6 +1,5 @@ package sqlancer.duckdb.test; -import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -8,21 +7,20 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.TestOracle; -import sqlancer.ast.newast.NewAliasNode; -import sqlancer.ast.newast.NewBinaryOperatorNode; -import sqlancer.ast.newast.NewFunctionNode; -import sqlancer.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.ast.newast.Node; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBAlias; +import sqlancer.duckdb.ast.DuckDBBinaryOperator; import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBFunction; import sqlancer.duckdb.ast.DuckDBSelect; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBAggregateFunction; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBBinaryArithmeticOperator; @@ -30,7 +28,8 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPostfixOperator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPrefixOperator; -public class DuckDBQueryPartitioningAggregateTester extends DuckDBQueryPartitioningBase implements TestOracle { +public class DuckDBQueryPartitioningAggregateTester extends DuckDBQueryPartitioningBase + implements TestOracle { private String firstResult; private String secondResult; @@ -48,24 +47,23 @@ public void check() throws SQLException { DuckDBAggregateFunction aggregateFunction = Randomly.fromOptions(DuckDBAggregateFunction.MAX, DuckDBAggregateFunction.MIN, DuckDBAggregateFunction.SUM, DuckDBAggregateFunction.COUNT, DuckDBAggregateFunction.AVG/* , DuckDBAggregateFunction.STDDEV_POP */); - NewFunctionNode aggregate = gen - .generateArgsForAggregate(aggregateFunction); - List> fetchColumns = new ArrayList<>(); + DuckDBFunction aggregate = gen.generateArgsForAggregate(aggregateFunction); + List fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { fetchColumns.add(gen.generateAggregate()); } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = DuckDBToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); secondResult = getAggregateResult(metamorphicQuery); - state.getState().queryString = "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult - + "\n-- " + secondResult; + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); if (firstResult == null && secondResult != null || firstResult != null && (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { @@ -77,19 +75,19 @@ public void check() throws SQLException { } - private String createMetamorphicUnionQuery(DuckDBSelect select, - NewFunctionNode aggregate, List> from) { + private String createMetamorphicUnionQuery(DuckDBSelect select, DuckDBFunction aggregate, + List from) { String metamorphicQuery; - Node whereClause = gen.generateExpression(); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DuckDBExpression whereClause = gen.generateExpression(); + DuckDBExpression negatedClause = new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(whereClause, DuckDBUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DuckDBExpression notNullClause = new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(whereClause, DuckDBUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + List mappedAggregate = mapped(aggregate); DuckDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DuckDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DuckDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate).toString() + " FROM ("; + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; metamorphicQuery += DuckDBToStringVisitor.asString(leftSelect) + " UNION ALL " + DuckDBToStringVisitor.asString(middleSelect) + " UNION ALL " + DuckDBToStringVisitor.asString(rightSelect); @@ -99,8 +97,8 @@ private String createMetamorphicUnionQuery(DuckDBSelect select, private String getAggregateResult(String queryString) throws SQLException { String resultString; - QueryAdapter q = new QueryAdapter(queryString, errors); - try (ResultSet result = q.executeAndGet(state)) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } @@ -119,7 +117,7 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped(NewFunctionNode aggregate) { + private List mapped(DuckDBFunction aggregate) { DuckDBCastOperation count; switch (aggregate.getFunc()) { case COUNT: @@ -128,21 +126,19 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DuckDBFunction sum = new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.SUM); - count = new DuckDBCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), + count = new DuckDBCastOperation(new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), new DuckDBCompositeDataType(DuckDBDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: - NewFunctionNode sumSquared = new NewFunctionNode<>( - Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DuckDBFunction sumSquared = new DuckDBFunction<>( + Arrays.asList(new DuckDBBinaryOperator(aggregate.getArgs().get(0), aggregate.getArgs().get(0), DuckDBBinaryArithmeticOperator.MULT)), DuckDBAggregateFunction.SUM); - count = new DuckDBCastOperation( - new NewFunctionNode(aggregate.getArgs(), - DuckDBAggregateFunction.COUNT), + count = new DuckDBCastOperation(new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), new DuckDBCompositeDataType(DuckDBDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DuckDBFunction avg = new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: @@ -150,11 +146,11 @@ private List> mapped(NewFunctionNode> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); + for (DuckDBExpression expr : originalAggregateArgs) { + args.add(new DuckDBAlias(expr, "agg" + i++)); } return args; } @@ -172,8 +168,8 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private DuckDBSelect getSelect(List aggregates, List from, + DuckDBExpression whereClause, List joinList) { DuckDBSelect leftSelect = new DuckDBSelect(); leftSelect.setFetchColumns(aggregates); leftSelect.setFromList(from); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index d27c2cabe..1e75be7b3 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -2,46 +2,36 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TestOracle; -import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.ast.newast.Node; -import sqlancer.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPostfixOperator; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPrefixOperator; -public class DuckDBQueryPartitioningBase implements TestOracle { - - final DuckDBGlobalState state; - final Set errors = new HashSet<>(); +public class DuckDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { DuckDBSchema s; DuckDBTables targetTables; DuckDBExpressionGenerator gen; DuckDBSelect select; - Node predicate; - Node negatedPredicate; - Node isNullPredicate; public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { - this.state = state; + super(state); DuckDBErrors.addExpressionErrors(errors); } @@ -50,33 +40,32 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new DuckDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new DuckDBSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new NewUnaryPrefixOperatorNode<>(predicate, DuckDBUnaryPrefixOperator.NOT); - isNullPredicate = new NewUnaryPostfixOperatorNode<>(predicate, DuckDBUnaryPostfixOperator.IS_NULL); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new DuckDBColumn("*", null, false, false))); + columns.add(new DuckDBColumnReference(new DuckDBColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); } return columns; } - Node generatePredicate() { - return gen.generateExpression(); + @Override + protected ExpressionGenerator getGen() { + return gen; } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java index 89b7ea602..70cd731d6 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java @@ -38,7 +38,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index 81b6f28dd..c40a63a18 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -7,12 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.ast.newast.ColumnReferenceNode; -import sqlancer.ast.newast.Node; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; public class DuckDBQueryPartitioningGroupByTester extends DuckDBQueryPartitioningBase { @@ -41,15 +39,13 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, ComparatorHelper::canonicalizeResultValue); } @Override - List> generateFetchColumns() { - List> columns = new ArrayList<>(); - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - return columns; + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index a6b625463..b0ff0a44c 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -7,14 +7,14 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.TestOracle; -import sqlancer.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBQueryPartitioningHavingTester extends DuckDBQueryPartitioningBase implements TestOracle { +public class DuckDBQueryPartitioningHavingTester extends DuckDBQueryPartitioningBase + implements TestOracle { public DuckDBQueryPartitioningHavingTester(DuckDBGlobalState state) { super(state); @@ -29,7 +29,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); @@ -46,16 +46,16 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, ComparatorHelper::canonicalizeResultValue); } @Override - Node generatePredicate() { + protected DuckDBExpression generatePredicate() { return gen.generateHavingClause(); } @Override - List> generateFetchColumns() { + List generateFetchColumns() { return Arrays.asList(gen.generateHavingClause()); } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java deleted file mode 100644 index 132c02a62..000000000 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.duckdb.test; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.duckdb.DuckDBErrors; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBToStringVisitor; - -public class DuckDBQueryPartitioningWhereTester extends DuckDBQueryPartitioningBase { - - public DuckDBQueryPartitioningWhereTester(DuckDBGlobalState state) { - super(state); - DuckDBErrors.addGroupByErrors(errors); - } - - @Override - public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = DuckDBToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = DuckDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = DuckDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = DuckDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - -} diff --git a/src/sqlancer/gen/AbstractGenerator.java b/src/sqlancer/gen/AbstractGenerator.java deleted file mode 100644 index 6ed0c68e9..000000000 --- a/src/sqlancer/gen/AbstractGenerator.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.gen; - -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; - -public abstract class AbstractGenerator { - - protected final Set errors = new HashSet<>(); - protected final StringBuilder sb = new StringBuilder(); - protected boolean canAffectSchema; - - public Query getQuery() { - buildStatement(); - return new QueryAdapter(sb.toString(), errors, canAffectSchema); - } - - public abstract void buildStatement(); - -} diff --git a/src/sqlancer/gen/AbstractInsertGenerator.java b/src/sqlancer/gen/AbstractInsertGenerator.java deleted file mode 100644 index e04def13b..000000000 --- a/src/sqlancer/gen/AbstractInsertGenerator.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.gen; - -import java.util.List; - -import sqlancer.Randomly; - -public abstract class AbstractInsertGenerator { - - protected StringBuilder sb = new StringBuilder(); - - protected void insertColumns(List columns) { - for (int nrRows = 0; nrRows < Randomly.smallNumber() + 1; nrRows++) { - if (nrRows != 0) { - sb.append(", "); - } - sb.append("("); - for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { - if (nrColumn != 0) { - sb.append(", "); - } - insertValue(columns.get(nrColumn)); - } - sb.append(")"); - } - } - - protected abstract void insertValue(C tiDBColumn); - -} diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java new file mode 100644 index 000000000..291ee99c2 --- /dev/null +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -0,0 +1,36 @@ +package sqlancer.h2; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Table; + +public final class H2DeleteGenerator extends AbstractDeleteGenerator { + + private final H2GlobalState globalState; + + private H2DeleteGenerator(H2GlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { + return new H2DeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + appendDeleteFromTable(table.getName()); + if (Randomly.getBoolean()) { + appendWhereClause(H2ToStringVisitor.asString( + new H2ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); + } + if (Randomly.getBoolean()) { + appendLimitClause(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); + } + H2Errors.addExpressionErrors(errors); + H2Errors.addDeleteErrors(errors); + } + +} diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java new file mode 100644 index 000000000..5846c1a88 --- /dev/null +++ b/src/sqlancer/h2/H2Errors.java @@ -0,0 +1,73 @@ +package sqlancer.h2; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class H2Errors { + + private H2Errors() { + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("NULL not allowed for column"); + errors.add("Unique index or primary key violation"); + errors.add("Data conversion error"); + errors.add("Generated column"); + errors.add("Value too long for column"); + errors.add("Referential integrity constraint violation"); + errors.add("Check constraint invalid"); + errors.add("Check constraint violation"); + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("java.lang.ArithmeticException: BigInteger would overflow supported range"); + errors.add("Value too long for column"); + errors.add("Numeric value out of range"); + errors.add("are not comparable"); + errors.add("Data conversion error converting"); + errors.add("Feature not supported"); + + errors.add("must be in the GROUP BY list"); + errors.add("must be in the result list in this case"); // ORDER BY + errors.add("Division by zero"); + + // regexp + errors.add("Unclosed group near index"); + errors.add("Error in LIKE ESCAPE"); + + // functions + errors.add("Invalid value" /* ... for parameter */); + + errors.add("String format error"); // STRINGDECODE + errors.add(/* precision */ "must be between"); // TRUNCATE_VALUE + errors.add("Cannot parse \"TIMESTAMP\" constant"); // TRUNCATE + errors.add("Invalid parameter count for \"TRUNC\", expected count: \"1\""); // TRUNCATE + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getDeleteErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("No default value is set for column"); // referential actions + errors.add("Referential integrity constraint violation"); + errors.add("NULL not allowed for column"); + return errors; + } + + public static void addDeleteErrors(ExpectedErrors errors) { + errors.addAll(getDeleteErrors()); + } + +} diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java new file mode 100644 index 000000000..3d816292c --- /dev/null +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -0,0 +1,391 @@ +package sqlancer.h2; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2CompositeDataType; +import sqlancer.h2.H2Schema.H2DataType; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.ast.H2BetweenOperation; +import sqlancer.h2.ast.H2BinaryOperation; +import sqlancer.h2.ast.H2CaseOperation; +import sqlancer.h2.ast.H2CastNode; +import sqlancer.h2.ast.H2ColumnReference; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2InOperation; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; +import sqlancer.h2.ast.H2TableReference; +import sqlancer.h2.ast.H2UnaryPostfixOperation; +import sqlancer.h2.ast.H2UnaryPrefixOperation; + +public class H2ExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { + + private final H2GlobalState globalState; + private List tables; + + public H2ExpressionGenerator(H2GlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + BINARY_COMPARISON, BINARY_LOGICAL, UNARY_POSTFIX, UNARY_PREFIX, IN, BETWEEN, CASE, BINARY_ARITHMETIC, CAST, + FUNCTION; + } + + @Override + protected H2Expression generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + Expression expr = Randomly.fromOptions(Expression.values()); + switch (expr) { + case BINARY_COMPARISON: + Operator op = H2BinaryComparisonOperator.getRandom(); + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = H2BinaryLogicalOperator.getRandom(); + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case UNARY_POSTFIX: + op = H2UnaryPostfixOperator.getRandom(); + return new H2UnaryPostfixOperation(generateExpression(depth + 1), op); + case UNARY_PREFIX: + return new H2UnaryPrefixOperation(generateExpression(depth + 1), H2UnaryPrefixOperator.getRandom()); + case IN: + return new H2InOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case BETWEEN: + return new H2BetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new H2CaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + case BINARY_ARITHMETIC: + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + H2BinaryArithmeticOperator.getRandom()); + case CAST: + return new H2CastNode(generateExpression(depth + 1), H2CompositeDataType.getRandom()); + case FUNCTION: + H2Function func = H2Function.getRandom(); + return new sqlancer.h2.ast.H2Function<>(generateExpressions(func.getNrArgs()), func); + default: + throw new AssertionError(); + } + } + + public enum H2Function { + + // numeric functions + ABS(1), // + ACOS(1), // + ASIN(1), // + ATAN(1), // + COS(1), // + COSH(1), // + COT(1), // + SIN(1), // + SINH(1), // + TAN(1), // + TANH(1), // + ATAN2(2), // + BITAND(2), // + BITGET(2), // + BITNOT(1), // + BITOR(2), // + BITXOR(2), // + LSHIFT(2), // + RSHIFT(2), // + MOD(2), // + CEILING(1), // + DEGREES(1), // + EXP(1), // + FLOOR(1), // + LN(1), // + LOG(2), // + LOG10(1), // + ORA_HASH(1), // + RADIANS(1), // + SQRT(1), // + PI(0), // + POWER(2), // + ROUND(2), // + ROUNDMAGIC(1), // + SIGN(1), // + TRUNCATE(2), // + COMPRESS(1), // + ZERO(0), // + // string functions + ASCII(1), // + BIT_LENGTH(1), // + LENGTH(1), // + OCTET_LENGTH(1), // + CHAR(1), // + CONCAT(2, true), // + CONCAT_WS(3, true), // + DIFFERENCE(2), // + HEXTORAW(1), // + RAWTOHEX(1), // + INSTR(3), // + INSERT(4), // + LOWER(1), // + UPPER(1), // + LEFT(2), // + RIGHT(2), // + LOCATE(3), // + POSITION(2), // + LTRIM(1), // + RTRIM(1), // + TRIM(1), // + REGEXP_REPLACE(3), // + REGEXP_LIKE(2), // + REPLACE(3), // + SOUNDEX(1), // + STRINGDECODE(1), // + STRINGENCODE(1), // + STRINGTOUTF8(1), // + SUBSTRING(2), // + UTF8TOSTRING(1), // + QUOTE_IDENT(1), // + XMLATTR(2), // + XMLNODE(1), // + XMLCOMMENT(1), // + XMLCDATA(1), // + XMLSTARTDOC(0), // + XMLTEXT(1), // + TRANSLATE(3), // + // TODO: time and date function + // systems functions + // TODO: array functions + CASEWHEN(3), // + COALESCE(1, true), // + CURRENT_SCHEMA(0), // + CURRENT_CATALOG(0), // + DATABASE_PATH(0), // + DECODE(3, true), // + GREATEST(2, true), // + IFNULL(2), // + LEAST(2, true), // + LOCK_MODE(0), // + LOCK_TIMEOUT(0), // + NULLIF(2), // + NVL2(3), // + READONLY(0), // + SESSION_ID(0), // + TRUNCATE_VALUE(3), // + USER(0); + // TODO JSON functions + + private int nrArgs; + private boolean isVariadic; + + H2Function(int nrArgs) { + this(nrArgs, false); + } + + H2Function(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static H2Function getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + } + + @Override + protected H2Expression generateColumn() { + return new H2ColumnReference(Randomly.fromList(columns)); + } + + @Override + public H2Expression generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return H2Constant.createNullConstant(); + } + switch (H2DataType.getRandom()) { + case INT: + return H2Constant.createIntConstant(globalState.getRandomly().getInteger()); + case BOOL: + return H2Constant.createBoolConstant(Randomly.getBoolean()); + case VARCHAR: + return H2Constant.createStringConstant(globalState.getRandomly().getString()); + case DOUBLE: + return H2Constant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BINARY: + return H2Constant.createBinaryConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(); + } + } + + public enum H2UnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"), IS_TRUE("IS TRUE"), IS_NOT_TRUE("IS NOT TRUE"), + IS_FALSE("IS FALSE"), IS_NOT_FALSE("IS NOT FALSE"), IS_UNKNOWN("IS NOT UNKNOWN"); + + private String textRepr; + + H2UnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static H2UnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum H2UnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private String textRepr; + + H2UnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static H2UnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum H2BinaryArithmeticOperator implements Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); + + private String textRepr; + + H2BinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum H2BinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum H2BinaryComparisonOperator implements Operator { + + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + IS_DISTINCT_FROM("IS DISTINCT FROM"), IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), LIKE("LIKE"), + NOT_LIKE("NOT LIKE"), REGEXP("REGEXP"), NOT_REGEXP("NOT REGEXP"); + + private String textRepr; + + H2BinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + @Override + public H2Expression negatePredicate(H2Expression predicate) { + return new H2UnaryPrefixOperation(predicate, H2UnaryPrefixOperator.NOT); + } + + @Override + public H2Expression isNull(H2Expression expr) { + return new H2UnaryPostfixOperation(expr, H2UnaryPostfixOperator.IS_NULL); + } + + @Override + public TLPWhereGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public H2Expression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public H2Select generateSelect() { + return new H2Select(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new H2TableReference(t)) + .collect(Collectors.toList()); + List joins = H2Join.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new H2TableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBoolean()) { + return List.of(new H2ColumnReference(new H2Column("*", null))); + } + return Randomly.nonEmptySubset(this.columns).stream().map(c -> new H2ColumnReference(c)) + .collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/h2/H2IndexGenerator.java b/src/sqlancer/h2/H2IndexGenerator.java new file mode 100644 index 000000000..b0d65e096 --- /dev/null +++ b/src/sqlancer/h2/H2IndexGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.h2; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Table; + +public class H2IndexGenerator { + + private final H2GlobalState globalState; + + public H2IndexGenerator(H2GlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { + return new H2IndexGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (Randomly.getBoolean()) { + sb.append("UNIQUE "); + errors.add("Unique index or primary key violation"); + } + if (Randomly.getBoolean()) { + sb.append("HASH "); + } + sb.append("INDEX IF NOT EXISTS "); + sb.append(globalState.getSchema().getFreeIndexName()); + sb.append(" ON "); + H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append('('); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(table.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(' '); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBoolean()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + sb.append(')'); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java new file mode 100644 index 000000000..3bd7552d8 --- /dev/null +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -0,0 +1,58 @@ +package sqlancer.h2; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; + +public class H2InsertGenerator extends AbstractInsertGenerator { + + private final H2GlobalState globalState; + private final H2ExpressionGenerator gen; + + public H2InsertGenerator(H2GlobalState globalState) { + this.globalState = globalState; + gen = new H2ExpressionGenerator(globalState); + } + + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { + return new H2InsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean mergeInto = false; // Randomly.getBooleanWithRatherLowProbability(); + if (mergeInto) { + sb.append("MERGE INTO "); + errors.add("Index \"PRIMARY_KEY_\" not found"); + errors.add("contains null values"); + errors.add("Valid MERGE INTO statement with at least one updatable column"); + } else { + sb.append("INSERT INTO "); + } + H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + appendColumnList(columns); + if (mergeInto && Randomly.getBoolean()) { + sb.append(" KEY("); + sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> c.getName()) + .collect(Collectors.joining(", "))); + sb.append(")"); + } + sb.append(" VALUES "); + insertColumns(columns); + H2Errors.addInsertErrors(errors); + H2Errors.addExpressionErrors(errors); // generated columns + } + + @Override + protected void insertValue(H2Column columnH2) { + sb.append(H2ToStringVisitor.asString(gen.generateConstant())); + } +} diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java new file mode 100644 index 000000000..985853e94 --- /dev/null +++ b/src/sqlancer/h2/H2Options.java @@ -0,0 +1,18 @@ +package sqlancer.h2; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(commandDescription = "H2") +public class H2Options implements DBMSSpecificOptions { + + @Override + public List getTestOracleFactory() { + return Arrays.asList(H2OracleFactory.TLP_WHERE); + } + +} diff --git a/src/sqlancer/h2/H2OracleFactory.java b/src/sqlancer/h2/H2OracleFactory.java new file mode 100644 index 000000000..fbbaade05 --- /dev/null +++ b/src/sqlancer/h2/H2OracleFactory.java @@ -0,0 +1,23 @@ +package sqlancer.h2; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; + +public enum H2OracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(H2Provider.H2GlobalState globalState) throws SQLException { + H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(H2Errors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }; + +} diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java new file mode 100644 index 000000000..7f01e6196 --- /dev/null +++ b/src/sqlancer/h2/H2Provider.java @@ -0,0 +1,115 @@ +package sqlancer.h2; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.h2.H2Provider.H2GlobalState; + +@AutoService(DatabaseProvider.class) +public class H2Provider extends SQLProviderAdapter { + + public H2Provider() { + super(H2GlobalState.class, H2Options.class); + } + + public enum Action implements AbstractAction { + + INSERT(H2InsertGenerator::getQuery), // + INDEX(H2IndexGenerator::getQuery), // + ANALYZE((g) -> new SQLQueryAdapter("ANALYZE")), // + CREATE_VIEW(H2ViewGenerator::getQuery), // + UPDATE(H2UpdateGenerator::getQuery), // + DELETE(H2DeleteGenerator::getQuery), // + SET(H2SetGenerator::getQuery); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(H2GlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(H2GlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case ANALYZE: + return r.getInteger(0, 5); + case INDEX: + case SET: + return r.getInteger(0, 5); + case CREATE_VIEW: + return r.getInteger(0, 2); + case UPDATE: + case DELETE: + return r.getInteger(0, 10); + default: + throw new AssertionError(a); + } + } + + public static class H2GlobalState extends SQLGlobalState { + + @Override + protected H2Schema readSchema() throws SQLException { + return H2Schema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(H2GlobalState globalState) throws Exception { + if (Randomly.getBoolean()) { + H2SetGenerator.getQuery(globalState).execute(globalState); + } + boolean success; + for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { + do { + SQLQueryAdapter qt = new H2TableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + H2Provider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(H2GlobalState globalState) throws SQLException { + String connectionString = "jdbc:h2:~/" + globalState.getDatabaseName() + ";DB_CLOSE_ON_EXIT=FALSE"; + Connection connection = DriverManager.getConnection(connectionString, "sa", ""); + connection.createStatement().execute("DROP ALL OBJECTS DELETE FILES"); + connection.close(); + connection = DriverManager.getConnection(connectionString, "sa", ""); + return new SQLConnection(connection); + } + + @Override + public String getDBMSName() { + return "h2"; + } + +} diff --git a/src/sqlancer/h2/H2RandomQuerySynthesizer.java b/src/sqlancer/h2/H2RandomQuerySynthesizer.java new file mode 100644 index 000000000..7390b87e7 --- /dev/null +++ b/src/sqlancer/h2/H2RandomQuerySynthesizer.java @@ -0,0 +1,60 @@ +package sqlancer.h2; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.H2Schema.H2Tables; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; +import sqlancer.h2.ast.H2TableReference; + +public final class H2RandomQuerySynthesizer { + + private H2RandomQuerySynthesizer() { + } + + public static H2Select generateSelect(H2GlobalState globalState, int nrColumns) { + H2Tables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(targetTables.getColumns()); + H2Select select = new H2Select(); + List columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + H2Expression expression = gen.generateExpression(); + columns.add(expression); + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new H2TableReference(t)) + .collect(Collectors.toList()); + List joins = H2Join.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + if (Randomly.getBoolean()) { + select.setLimitClause(H2Constant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(H2Constant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; + } + +} diff --git a/src/sqlancer/h2/H2Schema.java b/src/sqlancer/h2/H2Schema.java new file mode 100644 index 000000000..7a64ef56e --- /dev/null +++ b/src/sqlancer/h2/H2Schema.java @@ -0,0 +1,230 @@ +package sqlancer.h2; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Table; + +public class H2Schema extends AbstractSchema { + + public enum H2DataType { + + INT, BOOL, VARCHAR, DOUBLE, BINARY; + + public static H2DataType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public static class H2CompositeDataType { + + static final int NO_PRECISION = -1; + + private final H2DataType dataType; + private final int size; + private final int precision; + + public H2CompositeDataType(H2DataType dataType, int size, int precision) { + this.dataType = dataType; + this.size = size; + this.precision = precision; + } + + public H2DataType getPrimitiveDataType() { + return dataType; + } + + public static H2CompositeDataType getRandom() { + H2DataType primitiveType = Randomly.fromOptions(H2DataType.INT, H2DataType.BOOL, H2DataType.DOUBLE, + H2DataType.BINARY); + int size = -1; + int precision = NO_PRECISION; + switch (primitiveType) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case DOUBLE: + size = Randomly.fromOptions(4, 8); + if (Randomly.getBoolean()) { + if (size == 4) { + precision = (int) Randomly.getNotCachedInteger(1, 25); // TODO: documentation states 0 as lower + // bound + } else { + precision = (int) Randomly.getNotCachedInteger(25, 54); + } + } + break; + case VARCHAR: + case BINARY: + precision = (int) Randomly.getNotCachedInteger(0, Integer.MAX_VALUE); + break; + default: + break; + } + return new H2CompositeDataType(primitiveType, size, precision); + } + + @Override + public String toString() { + switch (dataType) { + case INT: + switch (size) { + case 1: + return "TINYINT"; + case 2: + return Randomly.fromOptions("SMALLINT", "INT2"); + case 4: + return Randomly.fromOptions("INT", "INTEGER", "MEDIUMINT", "INT4", "SIGNED"); + case 8: + return Randomly.fromOptions("BIGINT", "INT8"); + default: + throw new AssertionError(size); + } + case DOUBLE: + switch (size) { + case 4: + if (precision == NO_PRECISION) { + return Randomly.fromOptions("REAL", "FLOAT4"); + } else { + assert precision >= 0 && precision <= 24; + return String.format("FLOAT(%d)", precision); + } + case 8: + if (precision == NO_PRECISION) { + return Randomly.fromOptions("DOUBLE", "DOUBLE PRECISION", "FLOAT8", "FLOAT"); + } else { + assert precision >= 25 && precision <= 53; + return String.format("FLOAT(%d)", precision); + } + default: + throw new AssertionError(size); + } + case VARCHAR: + return /* String varCharType = */ Randomly.fromOptions("VARCHAR", "VARCHAR_IGNORECASE"); + // if (precision == NO_PRECISION) { + // return varCharType; + // } else { + // return String.format("%s(%d)", varCharType, precision); + // } + case BINARY: + return "BINARY"; + // return String.format("BINARY(%d)", precision); + default: + return dataType.toString(); + } + } + + } + + public static class H2Column extends AbstractTableColumn { + + public H2Column(String name, H2CompositeDataType columnType) { + super(name, null, columnType); + } + + } + + public static class H2Tables extends AbstractTables { + + public H2Tables(List tables) { + super(tables); + } + + } + + public H2Schema(List databaseTables) { + super(databaseTables); + } + + public H2Tables getRandomTableNonEmptyTables() { + return new H2Tables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public static class H2Table extends AbstractRelationalTable { + + public H2Table(String tableName, List columns) { + super(tableName, columns, Collections.emptyList(), tableName.startsWith("V")); + } + + } + + public static H2Schema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + H2Table t = new H2Table(tableName, databaseColumns); + for (H2Column c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new H2Schema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES")) { + while (rs.next()) { + tableNames.add(rs.getString("TABLE_NAME")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("SHOW COLUMNS FROM %s;", tableName))) { + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String columnType = rs.getString("TYPE"); + H2DataType primitiveType = getColumnType(columnType); + H2Column c = new H2Column(columnName, + new H2CompositeDataType(primitiveType, -1, -1 /* TODO: read size and precision */)); + columns.add(c); + } + } + } + return columns; + } + + private static H2DataType getColumnType(String columnType) { + if (columnType.startsWith("INTEGER") || columnType.startsWith("SMALLINT") || columnType.startsWith("TINYINT") + || columnType.startsWith("BIGINT")) { + return H2DataType.INT; + } else if (columnType.startsWith("BOOLEAN")) { + return H2DataType.BOOL; + } else if (columnType.startsWith("CHARACTER VARYING")) { + return H2DataType.VARCHAR; + } else if (columnType.startsWith("DOUBLE") || columnType.startsWith("DECFLOAT") || columnType.startsWith("REAL") + || columnType.startsWith("FLOAT")) { + return H2DataType.DOUBLE; + } else if (columnType.startsWith("NUMERIC")) { + return H2DataType.INT; + } else if (columnType.contentEquals("NULL")) { + return H2DataType.INT; // for a NULL view column + } else if (columnType.startsWith("BINARY")) { + return H2DataType.BINARY; + } else { + throw new AssertionError(columnType); + } + } + +} diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java new file mode 100644 index 000000000..12e3b39eb --- /dev/null +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -0,0 +1,61 @@ +package sqlancer.h2; + +import java.util.function.Function; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; + +public final class H2SetGenerator { + + private H2SetGenerator() { + } + + private enum Option { + + ALLOW_LITERALS((r) -> Randomly.fromOptions("ALL")), // + CACHE_SIZE((r) -> Randomly.getNotCachedInteger(0, 1000000)), // + BINARY_COLLATION((r) -> Randomly.fromOptions("SIGNED", "UNSIGNED")), // + UUID_COLLATION((r) -> Randomly.fromOptions("SIGNED", "UNSIGNED")), // + BUILTIN_ALIAS_OVERRIDE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // + COLLATION((r) -> Randomly.fromOptions("OFF", "ENGLISH", "ENGLISH STRENGTH PRIMARY", + "ENGLISH STRENGTH SECONDARY", "ENGLISH STRENGTH TERTIARY", "ENGLISH STRENGTH IDENTICAL")), // + DEFAULT_NULL_ORDERING((r) -> Randomly.fromOptions("LOW", "HIGH", "FIRST", "LAST")), // + DEFAULT_TABLE_TYPE((r) -> Randomly.fromOptions("MEMORY", "CACHED")), // + IGNORECASE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // + LAZY_QUERY_EXECUTION((r) -> Randomly.fromOptions(0, 1)), // + MAX_MEMORY_ROWS((r) -> Randomly.getNotCachedInteger(0, 100000)), // + MAX_MEMORY_UNDO((r) -> Randomly.getNotCachedInteger(0, 100000)), // + MAX_OPERATION_MEMORY((r) -> Randomly.getNotCachedInteger(0, 100000)), // + // MODE((r) -> Randomly.fromOptions("REGULAR", "DB2", "DERBY", "HSQLDB", "MSSQLSERVER", "ORACLE", + // "POSTGRESQL")); + OPTIMIZE_REUSE_RESULTS((r) -> Randomly.fromOptions(0, 1)), // + QUERY_STATISTICS((r) -> Randomly.fromOptions("TRUE", "FALSE")), // + QUERY_STATISTICS_MAX_ENTRIES((r) -> Randomly.getNotCachedInteger(0, 100000)), // + REFERENTIAL_INTEGRITY((r) -> Randomly.fromOptions("TRUE", "FALSE")); // + + private Function prod; + + Option(Function prod) { + this.prod = prod; + } + + public static Option getRandom() { + return Randomly.fromOptions(Option.values()); + } + } + + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { + StringBuilder sb = new StringBuilder(); + Option o = Option.getRandom(); + sb.append("SET "); + sb.append(o); + sb.append(" "); + sb.append(o.prod.apply(globalState.getRandomly())); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("Collation cannot be changed because there is a data table"); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/h2/H2TableGenerator.java b/src/sqlancer/h2/H2TableGenerator.java new file mode 100644 index 000000000..d0659983e --- /dev/null +++ b/src/sqlancer/h2/H2TableGenerator.java @@ -0,0 +1,115 @@ +package sqlancer.h2; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2CompositeDataType; +import sqlancer.h2.H2Schema.H2Table; + +public class H2TableGenerator { + + public SQLQueryAdapter getQuery(H2GlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("already exists"); + StringBuilder sb = new StringBuilder("CREATE TABLE "); + sb.append(globalState.getSchema().getFreeTableName()); + sb.append("("); + List columnNames = new ArrayList<>(); + for (int i = 0; i < Randomly.fromOptions(1, 2, 3); i++) { + columnNames.add("c" + i); + } + + for (int i = 0; i < columnNames.size(); i++) { + if (i != 0) { + sb.append(", "); + } + String c = columnNames.get(i); + sb.append(c); + sb.append(" "); + sb.append(H2CompositeDataType.getRandom()); + boolean generated = Randomly.getBooleanWithRatherLowProbability(); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } + if (Randomly.getBooleanWithRatherLowProbability() && !generated) { + sb.append(" DEFAULT "); + sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); + } + if (generated) { + sb.append(" AS ("); + List columns = columnNames.stream().filter(cName -> !cName.contentEquals(c)) + .map(c2 -> new H2Column(c2, null)).collect(Collectors.toList()); + H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(columns); + sb.append(H2ToStringVisitor.asString(gen.generateExpression())); + H2Errors.addExpressionErrors(errors); + errors.add("not found"); // generated column cycles + sb.append(')'); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" SELECTIVITY "); + sb.append(Randomly.getNotCachedInteger(0, 101)); + } + if (Randomly.getBooleanWithRatherLowProbability() && !generated) { + sb.append(" UNIQUE"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" CHECK "); + sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState) + .setColumns(columnNames.stream().map(c2 -> new H2Column(c2, null)).collect(Collectors.toList())) + .generateExpression())); + H2Errors.addExpressionErrors(errors); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(", PRIMARY KEY("); + sb.append(Randomly.nonEmptySubset(columnNames).stream().collect(Collectors.joining(", "))); + sb.append(")"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + List foreignKeyColumns = Randomly.nonEmptySubset(columnNames); + sb.append(", FOREIGN KEY("); + sb.append(foreignKeyColumns.stream().collect(Collectors.joining(", "))); + sb.append(')'); + List foreignTableCandidates = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> !t.isView()).collect(Collectors.toList()); + if (foreignTableCandidates.isEmpty()) { + throw new IgnoreMeException(); + } + H2Table foreignKeyTable = Randomly.fromList(foreignTableCandidates); + sb.append(" REFERENCES "); + sb.append(foreignKeyTable.getName()); + sb.append('('); + if (foreignKeyTable.getColumns().size() < foreignKeyColumns.size()) { + throw new IgnoreMeException(); + } + sb.append(foreignKeyTable.getRandomNonEmptyColumnSubset(foreignKeyColumns.size()).stream() + .map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(')'); + if (Randomly.getBoolean()) { + sb.append(" ON DELETE "); + addReferentialAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" ON UPDATE "); + addReferentialAction(sb); + } + errors.add("are not comparable"); + errors.add(" cannot be updatable by a referential constraint with"); // generated columns + errors.add("not found"); // Constraint "PRIMARY KEY | UNIQUE (C0)" not found; + } + sb.append(")"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void addReferentialAction(StringBuilder sb) { + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT", "NO ACTION", "SET DEFAULT", "SET NULL")); + } + +} diff --git a/src/sqlancer/h2/H2ToStringVisitor.java b/src/sqlancer/h2/H2ToStringVisitor.java new file mode 100644 index 000000000..ab18ab235 --- /dev/null +++ b/src/sqlancer/h2/H2ToStringVisitor.java @@ -0,0 +1,94 @@ +package sqlancer.h2; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.h2.ast.H2CastNode; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; + +public class H2ToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(H2Expression expr) { + if (expr instanceof H2Constant) { + visit((H2Constant) expr); + } else if (expr instanceof H2Select) { + visit((H2Select) expr); + } else if (expr instanceof H2Join) { + visit((H2Join) expr); + } else if (expr instanceof H2CastNode) { + visit((H2CastNode) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(H2Constant constant) { + sb.append(constant.toString()); + } + + private void visit(H2CastNode cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(')'); + } + + private void visit(H2Join join) { + visit((H2Expression) join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" JOIN "); + visit((H2Expression) join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + public void visit(H2Select select) { + sb.append("SELECT "); + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(H2Expression expr) { + H2ToStringVisitor visitor = new H2ToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java new file mode 100644 index 000000000..05e348038 --- /dev/null +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.h2; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; + +public final class H2UpdateGenerator extends AbstractUpdateGenerator { + + private final H2GlobalState globalState; + private H2ExpressionGenerator gen; + + private H2UpdateGenerator(H2GlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { + return new H2UpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + H2Errors.addInsertErrors(errors); + H2Errors.addDeleteErrors(errors); + if (Randomly.getBoolean()) { + appendWhereClause(H2ToStringVisitor.asString(gen.generateExpression())); + } + H2Errors.addExpressionErrors(errors); + } + + @Override + protected void updateValue(H2Column column) { + sb.append(H2ToStringVisitor.asString(gen.generateConstant())); + } + +} diff --git a/src/sqlancer/h2/H2ViewGenerator.java b/src/sqlancer/h2/H2ViewGenerator.java new file mode 100644 index 000000000..78b174025 --- /dev/null +++ b/src/sqlancer/h2/H2ViewGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.h2; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.h2.H2Provider.H2GlobalState; + +public final class H2ViewGenerator { + + private H2ViewGenerator() { + } + + public static SQLQueryAdapter getQuery(H2GlobalState globalState) { + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(H2ToStringVisitor.asString(H2RandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + H2Errors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/h2/ast/H2BetweenOperation.java b/src/sqlancer/h2/ast/H2BetweenOperation.java new file mode 100644 index 000000000..77921d91e --- /dev/null +++ b/src/sqlancer/h2/ast/H2BetweenOperation.java @@ -0,0 +1,9 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class H2BetweenOperation extends NewBetweenOperatorNode implements H2Expression { + public H2BetweenOperation(H2Expression left, H2Expression middle, H2Expression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/h2/ast/H2BinaryOperation.java b/src/sqlancer/h2/ast/H2BinaryOperation.java new file mode 100644 index 000000000..d523256fd --- /dev/null +++ b/src/sqlancer/h2/ast/H2BinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class H2BinaryOperation extends NewBinaryOperatorNode implements H2Expression { + public H2BinaryOperation(H2Expression left, H2Expression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/h2/ast/H2CaseOperation.java b/src/sqlancer/h2/ast/H2CaseOperation.java new file mode 100644 index 000000000..75e51fc4d --- /dev/null +++ b/src/sqlancer/h2/ast/H2CaseOperation.java @@ -0,0 +1,12 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class H2CaseOperation extends NewCaseOperatorNode implements H2Expression { + public H2CaseOperation(H2Expression switchCondition, List conditions, List expressions, + H2Expression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/h2/ast/H2CastNode.java b/src/sqlancer/h2/ast/H2CastNode.java new file mode 100644 index 000000000..5e7602311 --- /dev/null +++ b/src/sqlancer/h2/ast/H2CastNode.java @@ -0,0 +1,23 @@ +package sqlancer.h2.ast; + +import sqlancer.h2.H2Schema.H2CompositeDataType; + +public class H2CastNode implements H2Expression { + + private final H2Expression expression; + private final H2CompositeDataType type; + + public H2CastNode(H2Expression expression, H2CompositeDataType type) { + this.expression = expression; + this.type = type; + } + + public H2Expression getExpression() { + return expression; + } + + public H2CompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/h2/ast/H2ColumnReference.java b/src/sqlancer/h2/ast/H2ColumnReference.java new file mode 100644 index 000000000..5864f0b53 --- /dev/null +++ b/src/sqlancer/h2/ast/H2ColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.h2.H2Schema; + +public class H2ColumnReference extends ColumnReferenceNode implements H2Expression { + public H2ColumnReference(H2Schema.H2Column column) { + super(column); + } + +} diff --git a/src/sqlancer/h2/ast/H2Constant.java b/src/sqlancer/h2/ast/H2Constant.java new file mode 100644 index 000000000..923bcb4a2 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Constant.java @@ -0,0 +1,133 @@ +package sqlancer.h2.ast; + +public class H2Constant implements H2Expression { + + private H2Constant() { + } + + public static class H2NullConstant extends H2Constant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class H2IntConstant extends H2Constant { + + private final long value; + + public H2IntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static class H2BoolConstant extends H2Constant { + + private final boolean value; + + public H2BoolConstant(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static class H2StringConstant extends H2Constant { + + private final String value; + + public H2StringConstant(String value) { + this.value = value; + } + + @Override + public String toString() { + return String.format("'%s'", value.replace("'", "''")); + } + + } + + public static class H2DoubleConstant extends H2Constant { + + private final double value; + + public H2DoubleConstant(double value) { + this.value = value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "POWER(0, -1)"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "(-POWER(0, -1))"; + } else if (Double.compare(value, -0.0) == 0) { + return "(-CAST(0 AS DOUBLE))"; + } else if (Double.isNaN(value)) { + return "SQRT(-1)"; + } else { + return String.valueOf(value); + } + } + + } + + public static class H2BinaryConstant extends H2Constant { + + private String value; + + public H2BinaryConstant(long value) { + this.value = Long.toHexString(value); + if (this.value.length() % 2 == 1) { + this.value = '0' + this.value; // pad with leading zero if needed + } + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "X'" + value + "'"; + } + + } + + public static H2Expression createIntConstant(long val) { + return new H2IntConstant(val); + } + + public static H2Expression createNullConstant() { + return new H2NullConstant(); + } + + public static H2Expression createBoolConstant(boolean val) { + return new H2BoolConstant(val); + } + + public static H2Expression createStringConstant(String val) { + return new H2StringConstant(val); + } + + public static H2Expression createDoubleConstant(double val) { + return new H2DoubleConstant(val); + } + + public static H2Expression createBinaryConstant(long val) { + return new H2BinaryConstant(val); + } + +} diff --git a/src/sqlancer/h2/ast/H2Expression.java b/src/sqlancer/h2/ast/H2Expression.java new file mode 100644 index 000000000..afd114c91 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Expression.java @@ -0,0 +1,8 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.h2.H2Schema.H2Column; + +public interface H2Expression extends Expression { + +} diff --git a/src/sqlancer/h2/ast/H2Function.java b/src/sqlancer/h2/ast/H2Function.java new file mode 100644 index 000000000..9442d6db7 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Function.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class H2Function extends NewFunctionNode implements H2Expression { + public H2Function(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/h2/ast/H2InOperation.java b/src/sqlancer/h2/ast/H2InOperation.java new file mode 100644 index 000000000..1389fb18b --- /dev/null +++ b/src/sqlancer/h2/ast/H2InOperation.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class H2InOperation extends NewInOperatorNode implements H2Expression { + public H2InOperation(H2Expression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/h2/ast/H2Join.java b/src/sqlancer/h2/ast/H2Join.java new file mode 100644 index 000000000..6aafd1de9 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Join.java @@ -0,0 +1,105 @@ +package sqlancer.h2.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.h2.H2ExpressionGenerator; +import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; + +public class H2Join implements H2Expression, Join { + + private final H2TableReference leftTable; + private final H2TableReference rightTable; + private final JoinType joinType; + private H2Expression onCondition; + + public enum JoinType { + INNER, CROSS, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public H2Join(H2TableReference leftTable, H2TableReference rightTable, JoinType joinType, + H2Expression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public H2TableReference getLeftTable() { + return leftTable; + } + + public H2TableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public H2Expression getOnCondition() { + return onCondition; + } + + public static List getJoins(List tableList, H2GlobalState globalState) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + H2TableReference leftTable = tableList.remove(0); + H2TableReference rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + H2ExpressionGenerator joinGen = new H2ExpressionGenerator(globalState).setColumns(columns); + JoinType random = H2Join.JoinType.getRandom(); + switch (random) { + case INNER: + joinExpressions.add(H2Join.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + joinExpressions.add(H2Join.createNaturalJoin(leftTable, rightTable)); + break; + case LEFT: + joinExpressions.add(H2Join.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions.add(H2Join.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case CROSS: + joinExpressions.add(new H2Join(leftTable, rightTable, JoinType.CROSS, null)); + break; + default: + throw new AssertionError(random); + } + } + return joinExpressions; + } + + public static H2Join createRightOuterJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { + return new H2Join(left, right, JoinType.RIGHT, predicate); + } + + public static H2Join createLeftOuterJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { + return new H2Join(left, right, JoinType.LEFT, predicate); + } + + public static H2Join createInnerJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { + return new H2Join(left, right, JoinType.INNER, predicate); + } + + public static H2Join createNaturalJoin(H2TableReference left, H2TableReference right) { + return new H2Join(left, right, JoinType.NATURAL, null); + } + + @Override + public void setOnClause(H2Expression onClause) { + onCondition = onClause; + } + +} diff --git a/src/sqlancer/h2/ast/H2Select.java b/src/sqlancer/h2/ast/H2Select.java new file mode 100644 index 000000000..e2f6f8519 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Select.java @@ -0,0 +1,31 @@ +package sqlancer.h2.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.H2ToStringVisitor; + +public class H2Select extends SelectBase + implements H2Expression, Select { + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (H2Expression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (H2Join) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return H2ToStringVisitor.asString(this); + } +} diff --git a/src/sqlancer/h2/ast/H2TableReference.java b/src/sqlancer/h2/ast/H2TableReference.java new file mode 100644 index 000000000..459337c41 --- /dev/null +++ b/src/sqlancer/h2/ast/H2TableReference.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.h2.H2Schema; + +public class H2TableReference extends TableReferenceNode implements H2Expression { + public H2TableReference(H2Schema.H2Table table) { + super(table); + } +} diff --git a/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java b/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java new file mode 100644 index 000000000..5c43effbe --- /dev/null +++ b/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class H2UnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements H2Expression { + public H2UnaryPostfixOperation(H2Expression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java b/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java new file mode 100644 index 000000000..24085dcb6 --- /dev/null +++ b/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class H2UnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements H2Expression { + public H2UnaryPrefixOperation(H2Expression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/hive/HiveBugs.java b/src/sqlancer/hive/HiveBugs.java new file mode 100644 index 000000000..43d42ce0b --- /dev/null +++ b/src/sqlancer/hive/HiveBugs.java @@ -0,0 +1,41 @@ +package sqlancer.hive; + +// do not make the fields final to avoid warnings +public final class HiveBugs { + + // Incorrect IS NULL evaluation for negation of string concatenation involving column references. + // -(c || 'x') evaluates to NULL at runtime, but IS NULL incorrectly returns false. + // The optimizer's nullability inference for GenericUDFOPNegative does not account for + // runtime conversion failures producing NULL from non-null input. + // Reproduce: CREATE TABLE t(c DOUBLE); INSERT INTO t VALUES(1.0); + // SELECT (-(c || 'x')) IS NULL FROM t; -- returns false, expected true + // Affects: 4.0.1, 4.2.0 + public static boolean bugNegationNullability = true; + + // Non-boolean expressions (CAST to non-boolean, FLOOR, ROUND, arithmetic) silently + // return 0 rows for all three TLP partitions when used as WHERE predicates. + // Hive requires BOOLEAN in WHERE but does not error; instead it returns empty results. + // Reproduce: CREATE TABLE t(c INT); INSERT INTO t VALUES(1); + // SELECT * FROM t WHERE FLOOR(1); -- returns 0 rows, expected 1 + // Affects: 4.0.1, 4.2.0 + public static boolean bugNonBooleanWhereClause = true; + + // IN operator with boolean sub-expressions involving IS NULL evaluates incorrectly, + // returning 0 rows for all three TLP partitions. + // Reproduce: CREATE TABLE t(c BOOLEAN); INSERT INTO t VALUES(true),(false); + // SELECT * FROM t WHERE (c != c) IN ((false) IS NULL); -- returns 0, expected 2 + // Affects: 4.0.1, 4.2.0 + public static boolean bugInBooleanEvaluation = true; + + // BETWEEN with mixed boolean/numeric types has incorrect TLP evaluation. + // The IS NULL partition misses rows due to wrong nullability inference. + // Reproduce: CREATE TABLE t(c DOUBLE); INSERT INTO t VALUES(0.5),(1.5); + // SELECT * FROM t WHERE (c NOT IN (true)) NOT BETWEEN 0.01 AND c; + // -- TLP partitions lose rows + // Affects: 4.0.1, 4.2.0 + public static boolean bugBetweenMixedTypes = true; + + private HiveBugs() { + } + +} diff --git a/src/sqlancer/hive/HiveErrors.java b/src/sqlancer/hive/HiveErrors.java new file mode 100644 index 000000000..81b0be668 --- /dev/null +++ b/src/sqlancer/hive/HiveErrors.java @@ -0,0 +1,40 @@ +package sqlancer.hive; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class HiveErrors { + + private HiveErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot recognize input near"); + errors.add("Argument type mismatch"); + errors.add("Error while compiling statement"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Either CHECK or NOT NULL constraint violated!"); + errors.add("Error running query"); + errors.add("is different from preceding arguments"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} diff --git a/src/sqlancer/hive/HiveGlobalState.java b/src/sqlancer/hive/HiveGlobalState.java new file mode 100644 index 000000000..a362c1f88 --- /dev/null +++ b/src/sqlancer/hive/HiveGlobalState.java @@ -0,0 +1,11 @@ +package sqlancer.hive; + +import sqlancer.SQLGlobalState; + +public class HiveGlobalState extends SQLGlobalState { + + @Override + protected HiveSchema readSchema() throws Exception { + return HiveSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java new file mode 100644 index 000000000..ea55ac676 --- /dev/null +++ b/src/sqlancer/hive/HiveOptions.java @@ -0,0 +1,43 @@ +package sqlancer.hive; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.hive.gen.HiveExpressionGenerator; + +@Parameters(separators = "=", commandDescription = "Hive (default port: " + HiveOptions.DEFAULT_PORT + + ", default host: " + HiveOptions.DEFAULT_HOST + ")") +public class HiveOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 10000; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(HiveOracleFactory.TLPWhere); + + public enum HiveOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(HiveGlobalState globalState) throws SQLException { + HiveExpressionGenerator gen = new HiveExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HiveErrors.getExpressionErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }; + } + + @Override + public List getTestOracleFactory() { + return oracle; + } +} diff --git a/src/sqlancer/hive/HiveProvider.java b/src/sqlancer/hive/HiveProvider.java new file mode 100644 index 000000000..12798df93 --- /dev/null +++ b/src/sqlancer/hive/HiveProvider.java @@ -0,0 +1,119 @@ +package sqlancer.hive; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.hive.gen.HiveInsertGenerator; +import sqlancer.hive.gen.HiveTableGenerator; + +@AutoService(DatabaseProvider.class) +public class HiveProvider extends SQLProviderAdapter { + + public HiveProvider() { + super(HiveGlobalState.class, HiveOptions.class); + } + + public enum Action implements AbstractAction { + + INSERT(HiveInsertGenerator::getQuery); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(HiveGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(HiveGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(HiveGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + String tableName = globalState.getSchema().getFreeTableName(); + SQLQueryAdapter qt = HiveTableGenerator.generate(globalState, tableName); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + HiveProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(HiveGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = HiveOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = HiveOptions.DEFAULT_PORT; + } + + String databaseName = globalState.getDatabaseName(); + + String url = String.format("jdbc:hive2://%s:%d/%s", host, port, "default"); + Connection con = DriverManager.getConnection(url, username, password); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + con.close(); + con = DriverManager + .getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName, username, password)); + + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "hive"; + } +} diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java new file mode 100644 index 000000000..8733d5caa --- /dev/null +++ b/src/sqlancer/hive/HiveSchema.java @@ -0,0 +1,106 @@ +package sqlancer.hive; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.hive.HiveSchema.HiveTable; + +public class HiveSchema extends AbstractSchema { + + public enum HiveDataType { + + // TODO: support more types, e.g. TIMESTAMP, DATE, VARCHAR, CHAR, BINARY, ARRAY, MAP, STRUCT, UNIONTYPE... + STRING, INT, DOUBLE, BOOLEAN; + + public static HiveDataType getRandomType() { + return Randomly.fromList(Arrays.asList(values())); + } + } + + public static class HiveColumn extends AbstractTableColumn { + + public HiveColumn(String name, HiveTable table, HiveDataType type) { + super(name, table, type); + } + } + + public static class HiveTables extends AbstractTables { + + public HiveTables(List tables) { + super(tables); + } + } + + public static class HiveTable extends AbstractRelationalTable { + + public HiveTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + } + + public HiveSchema(List databaseTables) { + super(databaseTables); + } + + public static HiveSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + boolean isView = matchesViewName(tableName); + HiveTable t = new HiveTable(tableName, databaseColumns, isView); + for (HiveColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new HiveSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet tableRs = s.executeQuery("SHOW TABLES"); + while (tableRs.next()) { + String tableName = tableRs.getString(1); + tableNames.add(tableName); + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("DESCRIBE %s", tableName))) { + while (rs.next()) { + String columnName = rs.getString("col_name"); + String dataType = rs.getString("data_type"); + HiveColumn c = new HiveColumn(columnName, null, getColumnType(dataType.toUpperCase())); + columns.add(c); + } + } + } + return columns; + } + + private static HiveDataType getColumnType(String typeString) { + return HiveDataType.valueOf(typeString.toUpperCase()); + } + + public HiveTables getRandomTableNonEmptyTables() { + return new HiveTables(Randomly.nonEmptySubset(getDatabaseTables())); + } +} diff --git a/src/sqlancer/hive/HiveToStringVisitor.java b/src/sqlancer/hive/HiveToStringVisitor.java new file mode 100644 index 000000000..bdcd31eaf --- /dev/null +++ b/src/sqlancer/hive/HiveToStringVisitor.java @@ -0,0 +1,115 @@ +package sqlancer.hive; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hive.ast.HiveCastOperation; +import sqlancer.hive.ast.HiveConstant; +import sqlancer.hive.ast.HiveExpression; +import sqlancer.hive.ast.HiveJoin; +import sqlancer.hive.ast.HiveSelect; + +public class HiveToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(HiveExpression expr) { + if (expr instanceof HiveConstant) { + visit((HiveConstant) expr); + } else if (expr instanceof HiveSelect) { + visit((HiveSelect) expr); + } else if (expr instanceof HiveJoin) { + visit((HiveJoin) expr); + } else if (expr instanceof HiveCastOperation) { + visit((HiveCastOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(HiveConstant constant) { + sb.append(constant.toString()); + } + + private void visit(HiveSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(HiveJoin join) { + switch (join.getJoinType()) { + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException(); + } + visit((TableReferenceNode) join.getRightTable()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + private void visit(HiveCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(")"); + } + + public static String asString(HiveExpression expr) { + HiveToStringVisitor visitor = new HiveToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/hive/ast/HiveBetweenOperation.java b/src/sqlancer/hive/ast/HiveBetweenOperation.java new file mode 100644 index 000000000..26ec1d940 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class HiveBetweenOperation extends NewBetweenOperatorNode implements HiveExpression { + + public HiveBetweenOperation(HiveExpression left, HiveExpression middle, HiveExpression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/hive/ast/HiveBinaryOperation.java b/src/sqlancer/hive/ast/HiveBinaryOperation.java new file mode 100644 index 000000000..f74d117d4 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class HiveBinaryOperation extends NewBinaryOperatorNode implements HiveExpression { + + public HiveBinaryOperation(HiveExpression left, HiveExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/hive/ast/HiveCaseOperation.java b/src/sqlancer/hive/ast/HiveCaseOperation.java new file mode 100644 index 000000000..666070667 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveCaseOperation.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class HiveCaseOperation extends NewCaseOperatorNode implements HiveExpression { + + public HiveCaseOperation(HiveExpression switchCondition, List conditions, + List expressions, HiveExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/hive/ast/HiveCastOperation.java b/src/sqlancer/hive/ast/HiveCastOperation.java new file mode 100644 index 000000000..2d76ab4f2 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveCastOperation.java @@ -0,0 +1,25 @@ +package sqlancer.hive.ast; + +import sqlancer.hive.HiveSchema.HiveDataType; + +public class HiveCastOperation implements HiveExpression { + + private final HiveExpression expression; + private final HiveDataType type; + + public HiveCastOperation(HiveExpression expression, HiveDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public HiveExpression getExpression() { + return expression; + } + + public HiveDataType getType() { + return type; + } +} diff --git a/src/sqlancer/hive/ast/HiveColumnReference.java b/src/sqlancer/hive/ast/HiveColumnReference.java new file mode 100644 index 000000000..c3237955f --- /dev/null +++ b/src/sqlancer/hive/ast/HiveColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.hive.HiveSchema.HiveColumn; + +public class HiveColumnReference extends ColumnReferenceNode implements HiveExpression { + + public HiveColumnReference(HiveColumn column) { + super(column); + } +} diff --git a/src/sqlancer/hive/ast/HiveConstant.java b/src/sqlancer/hive/ast/HiveConstant.java new file mode 100644 index 000000000..7f89997d2 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveConstant.java @@ -0,0 +1,192 @@ +package sqlancer.hive.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public abstract class HiveConstant implements HiveExpression { + + public boolean isNull() { + return false; + } + + public static class HiveNullConstant extends HiveConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public String toString() { + return "NULL"; + } + } + + public static class HiveIntConstant extends HiveConstant { + + private final long value; + + public HiveIntConstant(long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class HiveDoubleConstant extends HiveConstant { + + private final double value; + + public HiveDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + } + + public static class HiveDecimalConstant extends HiveConstant { + + private final BigDecimal value; + + public HiveDecimalConstant(BigDecimal value) { + this.value = value; + } + + public BigDecimal getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class HiveTimestampConstant extends HiveConstant { + + private final String textRepr; + + public HiveTimestampConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + } + + public static class HiveDateConstant extends HiveConstant { + + private final String textRepr; + + public HiveDateConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + } + + public static class StringConstant extends HiveConstant { + + private final String value; + + public StringConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + } + + public static class HiveBooleanConstant extends HiveConstant { + + private final boolean value; + + public HiveBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static HiveConstant createNullConstant() { + return new HiveNullConstant(); + } + + public static HiveConstant createIntConstant(long value) { + return new HiveIntConstant(value); + } + + public static HiveConstant createDoubleConstant(double value) { + return new HiveDoubleConstant(value); + } + + public static HiveConstant createDecimalConstant(BigDecimal value) { + return new HiveDecimalConstant(value); + } + + public static HiveConstant createTimestampConstant(long value) { + return new HiveTimestampConstant(value); + } + + public static HiveConstant createDateConstant(long value) { + return new HiveDateConstant(value); + } + + public static HiveConstant createStringConstant(String value) { + return new StringConstant(value); + } + + public static HiveConstant createBooleanConstant(boolean value) { + return new HiveBooleanConstant(value); + } +} diff --git a/src/sqlancer/hive/ast/HiveExpression.java b/src/sqlancer/hive/ast/HiveExpression.java new file mode 100644 index 000000000..40842e181 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveExpression.java @@ -0,0 +1,7 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.hive.HiveSchema.HiveColumn; + +public interface HiveExpression extends Expression { +} diff --git a/src/sqlancer/hive/ast/HiveFunction.java b/src/sqlancer/hive/ast/HiveFunction.java new file mode 100644 index 000000000..b3a34ae25 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveFunction.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class HiveFunction extends NewFunctionNode implements HiveExpression { + + public HiveFunction(List args, F func) { + super(args, func); + } + +} diff --git a/src/sqlancer/hive/ast/HiveInOperation.java b/src/sqlancer/hive/ast/HiveInOperation.java new file mode 100644 index 000000000..601bf5e19 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class HiveInOperation extends NewInOperatorNode implements HiveExpression { + + public HiveInOperation(HiveExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/hive/ast/HiveJoin.java b/src/sqlancer/hive/ast/HiveJoin.java new file mode 100644 index 000000000..932ed9afa --- /dev/null +++ b/src/sqlancer/hive/ast/HiveJoin.java @@ -0,0 +1,48 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.Join; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; + +public class HiveJoin implements HiveExpression, Join { + + private final HiveTableReference leftTable; + private final HiveTableReference rightTable; + private final JoinType joinType; + private HiveExpression onClause; + + // TODO: test map-join optimization + + public enum JoinType { + INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, CROSS; + } + + public HiveJoin(HiveTableReference leftTable, HiveTableReference rightTable, JoinType joinType, + HiveExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onClause = onClause; + } + + public HiveTableReference getLeftTable() { + return leftTable; + } + + public HiveTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public HiveExpression getOnClause() { + return onClause; + } + + @Override + public void setOnClause(HiveExpression onClause) { + this.onClause = onClause; + } +} diff --git a/src/sqlancer/hive/ast/HiveOrderingTerm.java b/src/sqlancer/hive/ast/HiveOrderingTerm.java new file mode 100644 index 000000000..70fef52ad --- /dev/null +++ b/src/sqlancer/hive/ast/HiveOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class HiveOrderingTerm extends NewOrderingTerm implements HiveExpression { + + public HiveOrderingTerm(HiveExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java new file mode 100644 index 000000000..8a0eddc06 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -0,0 +1,41 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveToStringVisitor; + +public class HiveSelect extends SelectBase + implements Select, HiveExpression { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (HiveExpression) e) + .collect(java.util.stream.Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (HiveJoin) e).collect(java.util.stream.Collectors.toList()); + } + + @Override + public String asString() { + return HiveToStringVisitor.asString(this); + } + +} diff --git a/src/sqlancer/hive/ast/HiveTableReference.java b/src/sqlancer/hive/ast/HiveTableReference.java new file mode 100644 index 000000000..7d23b4895 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveTableReference.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hive.HiveSchema; + +public class HiveTableReference extends TableReferenceNode + implements HiveExpression { + + public HiveTableReference(HiveSchema.HiveTable table) { + super(table); + } + +} diff --git a/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java new file mode 100644 index 000000000..0461c5c73 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class HiveUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements HiveExpression { + + public HiveUnaryPostfixOperation(HiveExpression expr, Operator op) { + super(expr, op); + } + +} diff --git a/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java new file mode 100644 index 000000000..9fe286f82 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class HiveUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements HiveExpression { + + public HiveUnaryPrefixOperation(HiveExpression expr, Operator op) { + super(expr, op); + } + +} diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java new file mode 100644 index 000000000..92154873c --- /dev/null +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -0,0 +1,369 @@ +package sqlancer.hive.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.hive.HiveBugs; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveDataType; +import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.ast.HiveBetweenOperation; +import sqlancer.hive.ast.HiveBinaryOperation; +import sqlancer.hive.ast.HiveCaseOperation; +import sqlancer.hive.ast.HiveCastOperation; +import sqlancer.hive.ast.HiveColumnReference; +import sqlancer.hive.ast.HiveConstant; +import sqlancer.hive.ast.HiveExpression; +import sqlancer.hive.ast.HiveFunction; +import sqlancer.hive.ast.HiveInOperation; +import sqlancer.hive.ast.HiveJoin; +import sqlancer.hive.ast.HiveOrderingTerm; +import sqlancer.hive.ast.HiveSelect; +import sqlancer.hive.ast.HiveTableReference; +import sqlancer.hive.ast.HiveUnaryPostfixOperation; +import sqlancer.hive.ast.HiveUnaryPrefixOperation; + +public class HiveExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { + + private final HiveGlobalState globalState; + private List tables; + + private enum Expression { + // TODO: add or delete expressions. + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, CAST, FUNC, BETWEEN, IN, + CASE; + } + + public HiveExpressionGenerator(HiveGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public HiveExpression negatePredicate(HiveExpression predicate) { + return new HiveUnaryPrefixOperation(predicate, HiveUnaryPrefixOperator.NOT); + } + + @Override + public HiveExpression isNull(HiveExpression expr) { + return new HiveUnaryPostfixOperation(expr, HiveUnaryPostfixOperator.IS_NULL); + } + + @Override + protected HiveExpression generateExpression(int depth) { + // TODO: randomly cast some types like what PostgresExpressionGenerator does? + return generateExpressionInternal(depth); + } + + private HiveExpression generateExpressionInternal(int depth) throws AssertionError { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { + allowAggregates = false; // aggregate function calls cannot be nested + HiveAggregateFunction aggregate = HiveAggregateFunction.getRandom(); + return new HiveFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + if (HiveBugs.bugNonBooleanWhereClause) { + possibleOptions.remove(Expression.CAST); + possibleOptions.remove(Expression.FUNC); + possibleOptions.remove(Expression.BINARY_ARITHMETIC); + } + if (HiveBugs.bugInBooleanEvaluation) { + possibleOptions.remove(Expression.IN); + } + if (HiveBugs.bugBetweenMixedTypes) { + possibleOptions.remove(Expression.BETWEEN); + } + + Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case UNARY_PREFIX: + HiveUnaryPrefixOperator prefixOp = HiveUnaryPrefixOperator.getRandom(); + if (HiveBugs.bugNegationNullability + && (prefixOp == HiveUnaryPrefixOperator.MINUS || prefixOp == HiveUnaryPrefixOperator.PLUS)) { + throw new IgnoreMeException(); + } + return new HiveUnaryPrefixOperation(generateExpression(depth + 1), prefixOp); + case UNARY_POSTFIX: + return new HiveUnaryPostfixOperation(generateExpression(depth + 1), HiveUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = HiveBinaryComparisonOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = HiveExpressionGenerator.HiveBinaryLogicalOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); + case CAST: + return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); + case FUNC: + HiveFunc func = HiveFunc.getRandom(); + return new HiveFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new HiveBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new HiveInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new HiveCaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(expr); + } + } + + @Override + public HiveExpression generateConstant() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return HiveConstant.createNullConstant(); + } + HiveDataType[] values = HiveDataType.values(); + HiveDataType constantType = Randomly.fromOptions(values); + switch (constantType) { + case STRING: + return HiveConstant.createStringConstant(globalState.getRandomly().getString()); + case INT: + return HiveConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return HiveConstant.createBooleanConstant(Randomly.getBoolean()); + default: + throw new AssertionError(constantType); + } + } + + @Override + protected HiveExpression generateColumn() { + HiveColumn column = Randomly.fromList(columns); + return new HiveColumnReference(column); + } + + @Override + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (HiveExpression curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new HiveOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + } + + @Override + public HiveExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public HiveExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public HiveSelect generateSelect() { + return new HiveSelect(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new HiveTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean allowAggregates) { + if (Randomly.getBoolean()) { + return List.of(new HiveColumnReference(new HiveColumn("*", null, null))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new HiveColumnReference(c)) + .collect(Collectors.toList()); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + public enum HiveUnaryPrefixOperator implements Operator { + + // TODO: ~A (bitwise NOT) + NOT("NOT"), PLUS("+"), MINUS("-"); + + private String textRepr; + + HiveUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveUnaryPostfixOperator implements Operator { + + // TODO: A IS [NOT] (NULL|TRUE|FALSE)... + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + HiveUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryComparisonOperator implements Operator { + + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), REGEXP("RLIKE"); + + private String textRepr; + + HiveBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryLogicalOperator implements Operator { + + AND("AND"), OR("OR"); + + private String textRepr; + + HiveBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryArithmeticOperator implements Operator { + + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), BITWISE_AND("&"), BITWISE_OR("|"), + BITWISE_XOR("^"); + + private String textRepr; + + HiveBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveAggregateFunction { + COUNT(1), SUM(1), AVG(1), MIN(1), MAX(1), VARIANCE(1), VAR_SAMP(1), STDDEV_POP(1), STDDEV_SAMP(1), COVAR_POP(2), + COVAR_SAMP(2), CORR(2); + + private int nrArgs; + + HiveAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static HiveAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + // TODO: test all Hive default functions... + public enum HiveFunc { + + // mathematical functions + ROUND(2), FLOOR(1); + + // collection functions + + // date functions + + // string functions + + private int nrArgs; + private boolean isVariadic; + + HiveFunc(int nrArgs) { + this(nrArgs, false); + } + + HiveFunc(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static HiveFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + + } +} diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java new file mode 100644 index 000000000..8c23e4456 --- /dev/null +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -0,0 +1,51 @@ +package sqlancer.hive.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveToStringVisitor; + +public class HiveInsertGenerator extends AbstractInsertGenerator { + + private final HiveGlobalState globalState; + private final HiveExpressionGenerator gen; + + public HiveInsertGenerator(HiveGlobalState globalState) { + this.globalState = globalState; + this.gen = new HiveExpressionGenerator(globalState); + this.canonicalizeString = false; + } + + public static SQLQueryAdapter getQuery(HiveGlobalState globalState) { + return new HiveInsertGenerator(globalState).getStatement(); + } + + @Override + protected void insertValue(HiveColumn column) { + sb.append(HiveToStringVisitor.asString(gen.generateConstant())); + } + + @Override + public void buildStatement() { + // Inserting values into tables from SQL. + sb.append("INSERT INTO "); + HiveTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + + // TODO: specify the inserted partition + + sb.append(" VALUES "); + + // Values must be provided by every column in the Hive table. + // A value is either null or any valid SQL literal. + List columns = table.getColumns(); + insertColumns(columns); + + HiveErrors.addInsertErrors(errors); + } +} diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java new file mode 100644 index 000000000..c1c4db2bd --- /dev/null +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -0,0 +1,122 @@ +package sqlancer.hive.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveDataType; +import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveToStringVisitor; + +public class HiveTableGenerator { + + // TODO: support various file formats? e.g. JSONFILE, SEQUENCEFILE, TEXTFILE, RCFILE, ORC, PARQUET, AVRO. + + private enum ColumnConstraints { + PRIMARY_KEY_DISABLE, UNIQUE_DISABLE, NOT_NULL, DEFAULT, CHECK + // ENABLE_OR_DISABLE, NOVALIDATE, RELY_OR_NORELY + } + + private final HiveGlobalState globalState; + private final String tableName; + private final boolean allowPrimaryKey = Randomly.getBoolean(); + private final StringBuilder sb = new StringBuilder(); + private final HiveExpressionGenerator gen; + private final HiveTable table; + private final List columnsToBeAdded = new ArrayList<>(); + private boolean setPrimaryKey; + + public HiveTableGenerator(HiveGlobalState globalState, String tableName) { + this.tableName = tableName; + this.globalState = globalState; + this.table = new HiveTable(tableName, columnsToBeAdded, false); + this.gen = new HiveExpressionGenerator(globalState).setColumns(columnsToBeAdded); + } + + public static SQLQueryAdapter generate(HiveGlobalState globalState, String tableName) { + HiveTableGenerator generator = new HiveTableGenerator(globalState, tableName); + return generator.create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE TABLE "); + sb.append(globalState.getDatabaseName()); + sb.append("."); + sb.append(tableName); + sb.append(" ("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(i); + } + sb.append(")"); + + // TODO: implement PARTITION BY clause + // TODO: implement CLUSTERED BY, SKEWED BY clauses + // TODO: implement ROW FORMAT and STORED AS clauses + // TODO: randomly add some predefined TABLEPROPERTIES + // TODO: implement CTAS (AS clause) + + HiveErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + private void appendColumn(int columnId) { + String columnName = DBMSCommon.createColumnName(columnId); + sb.append(columnName); + sb.append(" "); + HiveDataType randType = HiveSchema.HiveDataType.getRandomType(); + sb.append(randType); + columnsToBeAdded.add(new HiveColumn(columnName, table, randType)); + appendColumnConstraint(); + } + + private void appendColumnConstraint() { + /* + * column_constraint_specification: : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK + * [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] + */ + if (Randomly.getBoolean()) { + // no column constraint + return; + } + + ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); + switch (constraint) { + case PRIMARY_KEY_DISABLE: + if (allowPrimaryKey && !setPrimaryKey) { + sb.append(" PRIMARY KEY DISABLE"); + setPrimaryKey = true; + } + break; + case UNIQUE_DISABLE: + sb.append(" UNIQUE DISABLE"); + break; + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + sb.append(" DEFAULT ("); + sb.append(HiveToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + case CHECK: + sb.append(" CHECK ("); + sb.append(HiveToStringVisitor.asString(gen.generateExpression())); + sb.append(")"); + break; + default: + throw new AssertionError(constraint); + } + } + +} diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java new file mode 100644 index 000000000..33995a0e1 --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -0,0 +1,43 @@ +package sqlancer.hsqldb; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class HSQLDBErrors { + + private HSQLDBErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("invalid datetime format"); + errors.add("invalid character value for cast"); + errors.add("invalid ORDER BY expression"); + errors.add("data type of expression is not boolean"); + errors.add("numeric value out of range"); + errors.add("incompatible data types in combination"); + errors.add("string data, right truncation"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getExpressionErrors()); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + +} diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java new file mode 100644 index 000000000..215f4f0fc --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -0,0 +1,21 @@ +package sqlancer.hsqldb; + +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(commandDescription = "hsqldb") +public class HSQLDBOptions implements DBMSSpecificOptions { + + @Parameter(names = "--oracle") + public List oracle = List.of(HSQLDBOracleFactory.WHERE, HSQLDBOracleFactory.NOREC); + + @Override + public List getTestOracleFactory() { + return oracle; + } + +} diff --git a/src/sqlancer/hsqldb/HSQLDBOracleFactory.java b/src/sqlancer/hsqldb/HSQLDBOracleFactory.java new file mode 100644 index 000000000..ed4fdf79c --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBOracleFactory.java @@ -0,0 +1,32 @@ +package sqlancer.hsqldb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +public enum HSQLDBOracleFactory implements OracleFactory { + WHERE { + @Override + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws SQLException { + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + NOREC { + @Override + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws Exception { + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + } +} diff --git a/src/sqlancer/hsqldb/HSQLDBProvider.java b/src/sqlancer/hsqldb/HSQLDBProvider.java new file mode 100644 index 000000000..0490f90c7 --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBProvider.java @@ -0,0 +1,110 @@ +package sqlancer.hsqldb; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.hsqldb.gen.HSQLDBInsertGenerator; +import sqlancer.hsqldb.gen.HSQLDBTableGenerator; +import sqlancer.hsqldb.gen.HSQLDBUpdateGenerator; + +@AutoService(DatabaseProvider.class) +public class HSQLDBProvider extends SQLProviderAdapter { + + private static final String HSQLDB = "hsqldb"; + + public HSQLDBProvider() { + super(HSQLDBGlobalState.class, HSQLDBOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(HSQLDBInsertGenerator::getQuery), UPDATE(HSQLDBUpdateGenerator::getQuery); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + @Override + public SQLConnection createDatabase(HSQLDBGlobalState globalState) throws Exception { + String databaseName = globalState.getDatabaseName(); + String url = "jdbc:hsqldb:file:" + databaseName; + MainOptions options = globalState.getOptions(); + Connection connection = DriverManager.getConnection(url, options.getUserName(), options.getPassword()); + // When a server instance is started, or when a connection is made to an in-process database, + // a new, empty database is created if no database exists at the given path. + try (Statement s = connection.createStatement()) { + s.execute("DROP SCHEMA PUBLIC CASCADE"); + s.execute("SET DATABASE SQL DOUBLE NAN FALSE"); + } + return new SQLConnection(connection); + } + + @Override + public String getDBMSName() { + return HSQLDB; + } + + @Override + public void generateDatabase(HSQLDBGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new HSQLDBTableGenerator().getQuery(globalState, null); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, + HSQLDBProvider.Action.values(), HSQLDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + private static int mapActions(HSQLDBProvider.HSQLDBGlobalState globalState, HSQLDBProvider.Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case UPDATE: + return r.getInteger(0, 10); + default: + throw new AssertionError(a); + } + } + + public static class HSQLDBGlobalState extends SQLGlobalState { + + @Override + protected HSQLDBSchema readSchema() throws SQLException { + return HSQLDBSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } +} diff --git a/src/sqlancer/hsqldb/HSQLDBSchema.java b/src/sqlancer/hsqldb/HSQLDBSchema.java new file mode 100644 index 000000000..2d41df83f --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBSchema.java @@ -0,0 +1,159 @@ +package sqlancer.hsqldb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; + +public class HSQLDBSchema extends AbstractSchema { + + public HSQLDBSchema(List databaseTables) { + super(databaseTables); + } + + public static HSQLDBSchema fromConnection(SQLConnection connection, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(connection); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(connection, tableName); + boolean isView = matchesViewName(tableName); + HSQLDBSchema.HSQLDBTable t = new HSQLDBSchema.HSQLDBTable(tableName, databaseColumns, isView); + for (HSQLDBSchema.HSQLDBColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new HSQLDBSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'PUBLIC';")) { + while (rs.next()) { + tableNames.add(rs.getString("TABLE_NAME")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + String sql = "SELECT COLUMN_NAME, DATA_TYPE, TYPE_NAME, COLUMN_SIZE FROM INFORMATION_SCHEMA.SYSTEM_COLUMNS WHERE TABLE_NAME = '%s';"; + try (ResultSet rs = s.executeQuery(String.format(sql, tableName))) { + while (rs.next()) { + HSQLDBDataType dataType = HSQLDBDataType.from(rs.getString("TYPE_NAME")); + HSQLDBCompositeDataType compositeDataType = new HSQLDBCompositeDataType(dataType, + rs.getInt("COLUMN_SIZE")); + HSQLDBColumn column = new HSQLDBColumn(rs.getString("COLUMN_NAME"), null, compositeDataType); + tableNames.add(column); + } + } + } + return tableNames; + } + + public static class HSQLDBTable + extends AbstractRelationalTable { + + public HSQLDBTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static class HSQLDBColumn + extends AbstractTableColumn { + + public HSQLDBColumn(String name, HSQLDBTable table, HSQLDBCompositeDataType type) { + super(name, table, type); + } + } + + public enum HSQLDBDataType { + + INTEGER, DOUBLE, BOOLEAN, CHAR, VARCHAR, BINARY, TIME, DATE, TIMESTAMP, NULL; + + public static HSQLDBSchema.HSQLDBDataType getRandomWithoutNull() { + HSQLDBSchema.HSQLDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == HSQLDBSchema.HSQLDBDataType.NULL); + return dt; + } + + public static HSQLDBDataType from(String typeName) { + for (HSQLDBDataType value : HSQLDBDataType.values()) { + if (value.name().equals(typeName)) { + return value; + } + } + return NULL; + } + } + + public static class HSQLDBCompositeDataType { + private final int size; + private final HSQLDBDataType type; + + public HSQLDBCompositeDataType(HSQLDBDataType type, int size) { + this.type = type; + this.size = size; + } + + public static HSQLDBCompositeDataType getRandomWithoutNull() { + HSQLDBSchema.HSQLDBDataType type = HSQLDBSchema.HSQLDBDataType.getRandomWithoutNull(); + return getRandomWithType(type); + } + + public static HSQLDBCompositeDataType getRandomWithType(HSQLDBSchema.HSQLDBDataType type) { + int size; + switch (type) { + case VARCHAR: + case CHAR: + case TIME: + case BINARY: + case TIMESTAMP: + size = Randomly.fromOptions(4, 6, 8); + break; + case BOOLEAN: + case INTEGER: + case DOUBLE: + // case UUID: + // case OTHER: + case DATE: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new HSQLDBSchema.HSQLDBCompositeDataType(type, size); + } + + public HSQLDBDataType getType() { + return type; + } + + public int getSize() { + return size; + } + } +} diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java new file mode 100644 index 000000000..99fc50687 --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -0,0 +1,89 @@ +package sqlancer.hsqldb; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBConstant; +import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; + +public class HSQLDBToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(HSQLDBExpression expr) { + if (expr instanceof HSQLDBConstant) { + visit((HSQLDBConstant) expr); + } else if (expr instanceof HSQLDBSelect) { + visit((HSQLDBSelect) expr); + } else if (expr instanceof HSQLDBJoin) { + visit((HSQLDBJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public static String asString(HSQLDBExpression expr) { + HSQLDBToStringVisitor visitor = new HSQLDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + private void visit(HSQLDBJoin join) { + visit((HSQLDBExpression) join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit((HSQLDBExpression) join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(HSQLDBConstant constant) { + sb.append(constant.toString()); + } + + private void visit(HSQLDBSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java new file mode 100644 index 000000000..b821a90e2 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class HSQLDBBinaryOperation extends NewBinaryOperatorNode implements HSQLDBExpression { + public HSQLDBBinaryOperation(HSQLDBExpression left, HSQLDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java new file mode 100644 index 000000000..cb579abac --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBColumnReference extends ColumnReferenceNode + implements HSQLDBExpression { + + public HSQLDBColumnReference(HSQLDBSchema.HSQLDBColumn column) { + super(column); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java new file mode 100644 index 000000000..67997d6bb --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java @@ -0,0 +1,227 @@ +package sqlancer.hsqldb.ast; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public class HSQLDBConstant implements HSQLDBExpression { + + private HSQLDBConstant() { + } + + public static class HSQLDBNullConstant extends HSQLDBConstant { + + @Override + public String toString() { + return "Null"; + } + + } + + public static class HSQLDBIntConstant extends HSQLDBConstant { + + private final int value; + + public HSQLDBIntConstant(long value) { + this.value = (int) value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class HSQLDBDoubleConstant extends HSQLDBConstant { + + private final double value; + + public HSQLDBDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "1.0e1/0.0e1"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-1.0e1/0.0e1"; + } + return String.valueOf(value); + } + + } + + public static class HSQLDBTextConstant extends HSQLDBConstant { + + private final String value; + + public HSQLDBTextConstant(String value) { + this.value = value; + } + + public HSQLDBTextConstant(String value, int size) { + this.value = value.substring(0, Math.min(value.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class HSQLDBBinaryConstant extends HSQLDBConstant { + + private final String value; + + public HSQLDBBinaryConstant(long value, int size) { + StringBuilder hex = new StringBuilder(Long.toHexString(value)); + if (hex.length() < 2) { + hex.append('0'); + } + this.value = hex.substring(0, Math.min(hex.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "X'" + value + "'"; + } + + } + + public static class HSQLDBDateConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + } + + public static class HSQLDBTimestampConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + } + + public static class HSQLDBTimeConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBTimeConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIME '%s'", textRepr); + } + + } + + public static class HSQLDBBooleanConstant extends HSQLDBConstant { + + private final boolean value; + + public HSQLDBBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static HSQLDBExpression createStringConstant(String text, int size) { + return new HSQLDBTextConstant(text, size); + } + + public static HSQLDBExpression createFloatConstant(double val) { + return new HSQLDBDoubleConstant(val); + } + + public static HSQLDBExpression createIntConstant(long val) { + return new HSQLDBIntConstant(val); + } + + public static HSQLDBExpression createNullConstant() { + return new HSQLDBNullConstant(); + } + + public static HSQLDBExpression createBooleanConstant(boolean val) { + return new HSQLDBBooleanConstant(val); + } + + public static HSQLDBExpression createDateConstant(long integer) { + return new HSQLDBDateConstant(integer); + } + + public static HSQLDBExpression createTimeConstant(long integer, int size) { + return new HSQLDBTimeConstant(integer); + } + + public static HSQLDBExpression createTimestampConstant(long integer, int size) { + return new HSQLDBTimestampConstant(integer); + } + + public static HSQLDBExpression createBinaryConstant(long nonCachedInteger, int size) { + return new HSQLDBBinaryConstant(nonCachedInteger, size); + } + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBExpression.java b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java new file mode 100644 index 000000000..d066b4359 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java @@ -0,0 +1,7 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; + +public interface HSQLDBExpression extends Expression { +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java new file mode 100644 index 000000000..a895e1b20 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -0,0 +1,91 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; + +public class HSQLDBJoin implements HSQLDBExpression, Join { + + private final HSQLDBTableReference leftTable; + private final HSQLDBTableReference rightTable; + private final JoinType joinType; + private HSQLDBExpression onCondition; + private OuterType outerType; + + public enum JoinType { + INNER, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + FULL, LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public HSQLDBJoin(HSQLDBTableReference leftTable, HSQLDBTableReference rightTable, JoinType joinType, + HSQLDBExpression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public HSQLDBTableReference getLeftTable() { + return leftTable; + } + + public HSQLDBTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public HSQLDBExpression getOnCondition() { + return onCondition; + } + + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public OuterType getOuterType() { + return outerType; + } + + public static HSQLDBJoin createRightOuterJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { + return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); + } + + public static HSQLDBJoin createLeftOuterJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { + return new HSQLDBJoin(left, right, JoinType.LEFT, predicate); + } + + public static HSQLDBJoin createInnerJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { + return new HSQLDBJoin(left, right, JoinType.INNER, predicate); + } + + public static HSQLDBJoin createNaturalJoin(HSQLDBTableReference left, HSQLDBTableReference right, + OuterType naturalJoinType) { + HSQLDBJoin join = new HSQLDBJoin(left, right, JoinType.NATURAL, null); + join.setOuterType(naturalJoinType); + return join; + } + + @Override + public void setOnClause(HSQLDBExpression onClause) { + onCondition = onClause; + } + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java new file mode 100644 index 000000000..d58275d1e --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java @@ -0,0 +1,41 @@ +package sqlancer.hsqldb.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.HSQLDBToStringVisitor; + +public class HSQLDBSelect extends SelectBase + implements HSQLDBExpression, Select { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (HSQLDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (HSQLDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return HSQLDBToStringVisitor.asString(this); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java b/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java new file mode 100644 index 000000000..010a921a7 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBTableReference extends TableReferenceNode + implements HSQLDBExpression { + public HSQLDBTableReference(HSQLDBSchema.HSQLDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java new file mode 100644 index 000000000..5ebca87fa --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java @@ -0,0 +1,57 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements HSQLDBExpression { + + public HSQLDBUnaryPostfixOperation(HSQLDBExpression expr, HSQLDBUnaryPostfixOperator op) { + super(expr, op); + } + + public enum HSQLDBUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public HSQLDBSchema.HSQLDBDataType[] getInputDataTypes() { + return HSQLDBSchema.HSQLDBDataType.values(); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public HSQLDBSchema.HSQLDBDataType[] getInputDataTypes() { + return HSQLDBSchema.HSQLDBDataType.values(); + } + }; + + private final String textRepresentations; + + HSQLDBUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static HSQLDBUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract HSQLDBSchema.HSQLDBDataType[] getInputDataTypes(); + + } + + public HSQLDBExpression getExpression() { + return getExpr(); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java new file mode 100644 index 000000000..44b2beece --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java @@ -0,0 +1,54 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements HSQLDBExpression { + + public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, HSQLDBExpression expression) { + super(expression, operation); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + + public enum HSQLDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", HSQLDBSchema.HSQLDBDataType.BOOLEAN, HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.BOOLEAN; + } + }, + + UNARY_PLUS("+", HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.INTEGER; + } + }, + UNARY_MINUS("-", HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.INTEGER; + } + }; + + private String textRepresentation; + + HSQLDBUnaryPrefixOperator(String textRepresentation, HSQLDBSchema.HSQLDBDataType... dataTypes) { + this.textRepresentation = textRepresentation; + } + + public abstract HSQLDBSchema.HSQLDBDataType getExpressionType(); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java new file mode 100644 index 000000000..be73d1cb1 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -0,0 +1,302 @@ +package sqlancer.hsqldb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.ast.HSQLDBBinaryOperation; +import sqlancer.hsqldb.ast.HSQLDBColumnReference; +import sqlancer.hsqldb.ast.HSQLDBConstant; +import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.ast.HSQLDBTableReference; +import sqlancer.hsqldb.ast.HSQLDBUnaryPostfixOperation; +import sqlancer.hsqldb.ast.HSQLDBUnaryPrefixOperation; + +public final class HSQLDBExpressionGenerator extends + TypedExpressionGenerator + implements NoRECGenerator, + TLPWhereGenerator { + + List tables; + + private enum Expression { + BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC; + } + + HSQLDBProvider.HSQLDBGlobalState hsqldbGlobalState; + + public HSQLDBExpressionGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.hsqldbGlobalState = globalState; + } + + @Override + public HSQLDBExpression generatePredicate() { + return generateExpression( + HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithType(HSQLDBSchema.HSQLDBDataType.BOOLEAN)); + } + + @Override + public HSQLDBExpression negatePredicate(HSQLDBExpression predicate) { + return new HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator.NOT, predicate); + } + + @Override + public HSQLDBExpression isNull(HSQLDBExpression expr) { + return new HSQLDBUnaryPostfixOperation(expr, HSQLDBUnaryPostfixOperation.HSQLDBUnaryPostfixOperator.IS_NULL); + } + + @Override + public HSQLDBExpression generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { + switch (type.getType()) { + case NULL: + return HSQLDBConstant.createNullConstant(); + case CHAR: + return HSQLDBConstant.HSQLDBTextConstant + .createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); + case VARCHAR: + return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getString(), + type.getSize()); + case TIME: + return HSQLDBConstant.createTimeConstant( + hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); + case TIMESTAMP: + return HSQLDBConstant.createTimestampConstant( + hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); + + case INTEGER: + return HSQLDBConstant.HSQLDBIntConstant.createIntConstant(Randomly.getNonCachedInteger()); + case DOUBLE: + return HSQLDBConstant.HSQLDBDoubleConstant.createFloatConstant(hsqldbGlobalState.getRandomly().getDouble()); + case BOOLEAN: + return HSQLDBConstant.HSQLDBBooleanConstant.createBooleanConstant(Randomly.getBoolean()); + case DATE: + return HSQLDBConstant + .createDateConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis())); + case BINARY: + return HSQLDBConstant.createBinaryConstant(Randomly.getNonCachedInteger(), type.getSize()); + default: + throw new AssertionError("Unknown type: " + type); + } + } + + @Override + protected HSQLDBExpression generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { + if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithSmallProbability()) { + return generateLeafNode(type); + } + + List possibleOptions = new ArrayList<>( + Arrays.asList(HSQLDBExpressionGenerator.Expression.values())); + + HSQLDBExpressionGenerator.Expression expr = Randomly.fromList(possibleOptions); + BinaryOperatorNode.Operator op; + switch (expr) { + case BINARY_LOGICAL: + case BINARY_ARITHMETIC: + op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); + break; + case BINARY_COMPARISON: + op = HSQLDBDBBinaryComparisonOperator.getRandom(); + break; + default: + throw new AssertionError(); + } + + return new HSQLDBBinaryOperation(generateExpression(type, depth + 1), generateExpression(type, depth + 1), op); + + } + + @Override + protected HSQLDBExpression generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { + HSQLDBSchema.HSQLDBColumn column = Randomly + .fromList(columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList())); + return new HSQLDBColumnReference(column); + } + + @Override + protected HSQLDBSchema.HSQLDBCompositeDataType getRandomType() { + return HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull(); + } + + @Override + protected boolean canGenerateColumnOfType(HSQLDBSchema.HSQLDBCompositeDataType type) { + return columns.stream().anyMatch(c -> c.getType() == type); + } + + public enum HSQLDBBinaryLogicalOperator implements BinaryOperatorNode.Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum HSQLDBDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); + + private String textRepr; + + HSQLDBDBBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum HSQLDBDBBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); + + private String textRepr; + + HSQLDBDBBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + @Override + public List generateOrderBys() { + List expressions = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + ArrayList hsqldbColumns = new ArrayList<>(columns); + for (int i = 0; i < nr && !hsqldbColumns.isEmpty(); i++) { + HSQLDBSchema.HSQLDBColumn randomColumn = Randomly.fromList(hsqldbColumns); + HSQLDBColumnReference columnReference = new HSQLDBColumnReference(randomColumn); + hsqldbColumns.remove(randomColumn); + expressions.add(columnReference); + } + return expressions; + } + + @Override + public HSQLDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public HSQLDBExpression generateBooleanExpression() { + return generatePredicate(); + } + + @Override + public HSQLDBSelect generateSelect() { + return new HSQLDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinExpressions = new ArrayList<>(); + while (tables.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + HSQLDBTable leftTable = tables.remove(0); + HSQLDBTable rightTable = tables.remove(0); + List columns = new ArrayList<>(leftTable.getColumns()); + columns.addAll(rightTable.getColumns()); + HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(hsqldbGlobalState).setColumns(columns); + HSQLDBTableReference leftTableRef = new HSQLDBTableReference(leftTable); + HSQLDBTableReference rightTableRef = new HSQLDBTableReference(rightTable); + switch (HSQLDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case NATURAL: + joinExpressions.add( + HSQLDBJoin.createNaturalJoin(leftTableRef, rightTableRef, HSQLDBJoin.OuterType.getRandom())); + break; + case LEFT: + joinExpressions.add(HSQLDBJoin.createLeftOuterJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case RIGHT: + joinExpressions.add(HSQLDBJoin.createRightOuterJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new HSQLDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(HSQLDBSelect select, HSQLDBExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + HSQLDBColumn aggr = new HSQLDBColumn("COUNT(*)", null, null); + select.setFetchColumns(List.of(new HSQLDBColumnReference(aggr))); + } else { + List allColumns = columns.stream().map((c) -> new HSQLDBColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(HSQLDBSelect select, HSQLDBExpression whereCondition) { + HSQLDBColumn c = new HSQLDBColumn("COUNT(*) as count", null, null); + select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); + select.setWhereClause(null); + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new HSQLDBColumnReference(new HSQLDBSchema.HSQLDBColumn("*", null, null))); + } + return Randomly + .nonEmptySubset(columns.stream().map(c -> new HSQLDBColumnReference(c)).collect(Collectors.toList())); + } +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java new file mode 100644 index 000000000..00d99be38 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -0,0 +1,38 @@ +package sqlancer.hsqldb.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBExpression; + +public class HSQLDBInsertGenerator extends AbstractInsertGenerator { + + private final HSQLDBProvider.HSQLDBGlobalState globalState; + + public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { + return new HSQLDBInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + buildInsertInto(table.getName(), columns); + } + + @Override + protected void insertValue(HSQLDBSchema.HSQLDBColumn column) { + HSQLDBExpression expression = new HSQLDBExpressionGenerator(globalState).generateConstant(column.getType()); + String s = HSQLDBToStringVisitor.asString(expression); + sb.append(s); + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java new file mode 100644 index 000000000..30249e780 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java @@ -0,0 +1,62 @@ +package sqlancer.hsqldb.gen; + +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBTableGenerator extends AbstractTableGenerator { + + private HSQLDBProvider.HSQLDBGlobalState globalState; + private String tableName; + + public HSQLDBTableGenerator() { + this.canAffectSchema = true; + } + + public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @Nullable String tableName) { + this.globalState = globalState; + this.tableName = tableName; + return getStatement(); + } + + @Override + public void buildStatement() { + String name = tableName; + if (name == null) { + name = globalState.getSchema().getFreeTableName(); + } + appendCreateTable(name, Randomly.getBoolean()); + appendColumnDefinitions(getNewColumns()); + sb.append(";"); + } + + @Override + protected void appendColumnDefinition(HSQLDBSchema.HSQLDBColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType().getType().name()); + if (column.getType().getSize() > 0) { + // Cannot specify size for non composite data types + sb.append("("); + sb.append(column.getType().getSize()); + sb.append(")"); + } + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + HSQLDBSchema.HSQLDBCompositeDataType columnType = HSQLDBSchema.HSQLDBCompositeDataType + .getRandomWithoutNull(); + columns.add(new HSQLDBSchema.HSQLDBColumn(columnName, null, columnType)); + } + return columns; + } +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java new file mode 100644 index 000000000..2e6081df4 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -0,0 +1,54 @@ +package sqlancer.hsqldb.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBErrors; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBCompositeDataType; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBDataType; +import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBExpression; + +public final class HSQLDBUpdateGenerator extends AbstractUpdateGenerator { + + private final HSQLDBProvider.HSQLDBGlobalState globalState; + private HSQLDBExpressionGenerator gen; + + private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { + return new HSQLDBUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + if (Randomly.getBooleanWithSmallProbability()) { + appendWhereClause(HSQLDBToStringVisitor.asString( + gen.generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)))); + errors.add("data type of expression is not boolean"); + HSQLDBErrors.addExpressionErrors(errors); + } + } + + @Override + protected void updateValue(HSQLDBColumn column) { + HSQLDBExpression expr; + expr = gen.generateConstant(column.getType()); + sb.append(HSQLDBToStringVisitor.asString(expr)); + } + +} diff --git a/src/sqlancer/mariadb/MariaDBBugs.java b/src/sqlancer/mariadb/MariaDBBugs.java index 7a568b8fe..5a1bfa5cc 100644 --- a/src/sqlancer/mariadb/MariaDBBugs.java +++ b/src/sqlancer/mariadb/MariaDBBugs.java @@ -5,6 +5,30 @@ public final class MariaDBBugs { // https://jira.mariadb.org/browse/MDEV-21058 public static boolean bug21058 = true; + // https://jira.mariadb.org/browse/MDEV-32076 + public static boolean bug32076 = true; + + // https://jira.mariadb.org/browse/MDEV-32099 + public static boolean bug32099 = true; + + // https://jira.mariadb.org/browse/MDEV-32105 + public static boolean bug32105 = true; + + // https://jira.mariadb.org/browse/MDEV-32106 + public static boolean bug32106 = true; + + // https://jira.mariadb.org/browse/MDEV-32107 + public static boolean bug32107 = true; + + // https://jira.mariadb.org/browse/MDEV-32108 + public static boolean bug32108 = true; + + // https://jira.mariadb.org/browse/MDEV-32143 + public static boolean bug32143 = true; + + // https://jira.mariadb.org/browse/MDEV-33893 + public static boolean bug33893 = true; + private MariaDBBugs() { } diff --git a/src/sqlancer/mariadb/MariaDBErrors.java b/src/sqlancer/mariadb/MariaDBErrors.java index f1e2f581f..b84deba02 100644 --- a/src/sqlancer/mariadb/MariaDBErrors.java +++ b/src/sqlancer/mariadb/MariaDBErrors.java @@ -1,13 +1,55 @@ package sqlancer.mariadb; +import java.util.ArrayList; import java.util.List; +import sqlancer.common.query.ExpectedErrors; + public final class MariaDBErrors { private MariaDBErrors() { } - public static void addInsertErrors(List errors) { + public static List getCommonErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("is out of range"); + // regex + errors.add("unmatched parentheses"); + errors.add("nothing to repeat at offset"); + errors.add("missing )"); + errors.add("missing terminating ]"); + errors.add("range out of order in character class"); + errors.add("unrecognized character after "); + errors.add("Got error '(*VERB) not recognized or malformed"); + errors.add("must be followed by"); + errors.add("malformed number or name after"); + errors.add("digit expected after"); + errors.add("Regex error"); + errors.add("Lock wait timeout exceeded"); + + return errors; + } + + public static void addCommonErrors(ExpectedErrors errors) { + errors.add("is out of range"); + // regex + errors.add("unmatched parentheses"); + errors.add("nothing to repeat at offset"); + errors.add("missing )"); + errors.add("missing terminating ]"); + errors.add("range out of order in character class"); + errors.add("unrecognized character after "); + errors.add("Got error '(*VERB) not recognized or malformed"); + errors.add("must be followed by"); + errors.add("malformed number or name after"); + errors.add("digit expected after"); + errors.add("Regex error"); + errors.add("Lock wait timeout exceeded"); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); errors.add("Out of range"); errors.add("Duplicate entry"); // violates UNIQUE constraint errors.add("cannot be null"); // violates NOT NULL constraint @@ -17,6 +59,11 @@ public static void addInsertErrors(List errors) { errors.add("The value specified for generated column"); // trying to insert into a generated column errors.add("Incorrect double value"); errors.add("Incorrect string value"); + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 6290ddf66..b1e7f807e 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -1,8 +1,25 @@ package sqlancer.mariadb; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -@Parameters -public class MariaDBOptions { +import sqlancer.DBMSSpecificOptions; + +@Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT + + ", default host: " + MariaDBOptions.DEFAULT_HOST + ")") +public class MariaDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3306; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(MariaDBOracleFactory.NOREC); + + @Override + public List getTestOracleFactory() { + return oracles; + } } diff --git a/src/sqlancer/mariadb/MariaDBOracleFactory.java b/src/sqlancer/mariadb/MariaDBOracleFactory.java new file mode 100644 index 000000000..549da27c8 --- /dev/null +++ b/src/sqlancer/mariadb/MariaDBOracleFactory.java @@ -0,0 +1,36 @@ +package sqlancer.mariadb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; +import sqlancer.mariadb.oracle.MariaDBDQPOracle; + +public enum MariaDBOracleFactory implements OracleFactory { + + NOREC { + @Override + public TestOracle create(MariaDBProvider.MariaDBGlobalState globalState) + throws SQLException { + MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MariaDBErrors.getCommonErrors()) + .with("is out of range").with("unmatched parentheses").with("nothing to repeat at offset") + .with("missing )").with("missing terminating ]").with("range out of order in character class") + .with("unrecognized character after ").with("Got error '(*VERB) not recognized or malformed") + .with("must be followed by").with("malformed number or name after").with("digit expected after") + .with("Could not create a join buffer").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + DQP { + @Override + public TestOracle create(MariaDBProvider.MariaDBGlobalState globalState) + throws SQLException { + return new MariaDBDQPOracle(globalState); + } + } +} diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 985b1375a..a9737f549 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -7,17 +7,19 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.GlobalState; +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; -import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.StateToReproduce; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.gen.MariaDBDeleteGenerator; import sqlancer.mariadb.gen.MariaDBIndexGenerator; import sqlancer.mariadb.gen.MariaDBInsertGenerator; import sqlancer.mariadb.gen.MariaDBSetGenerator; @@ -25,14 +27,11 @@ import sqlancer.mariadb.gen.MariaDBTableGenerator; import sqlancer.mariadb.gen.MariaDBTruncateGenerator; import sqlancer.mariadb.gen.MariaDBUpdateGenerator; -import sqlancer.mariadb.oracle.MariaDBNoRECOracle; -import sqlancer.sqlite3.gen.SQLite3Common; -public class MariaDBProvider extends ProviderAdapter { +@AutoService(DatabaseProvider.class) +public class MariaDBProvider extends SQLProviderAdapter { public static final int MAX_EXPRESSION_DEPTH = 3; - private final Randomly r = new Randomly(); - private String databaseName; public MariaDBProvider() { super(MariaDBGlobalState.class, MariaDBOptions.class); @@ -49,31 +48,18 @@ enum Action { SET, // TRUNCATE, // UPDATE, // + DELETE, } @Override - public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLException { - this.databaseName = globalState.getDatabaseName(); + public void generateDatabase(MariaDBGlobalState globalState) throws Exception { MainOptions options = globalState.getOptions(); - Connection con = globalState.getConnection(); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); - QueryManager manager = globalState.getManager(); - MariaDBSchema newSchema = MariaDBSchema.fromConnection(con, databaseName); - globalState.setSchema(newSchema); - if (options.logEachSelect()) { - logger.writeCurrent(state); - } - while (newSchema.getDatabaseTables().size() < Randomly.smallNumber() + 1) { - String tableName = SQLite3Common.createTableName(newSchema.getDatabaseTables().size()); - Query createTable = MariaDBTableGenerator.generate(tableName, r, newSchema); - if (options.logEachSelect()) { - logger.writeCurrent(createTable.getQueryString()); - } - manager.execute(createTable); - newSchema = MariaDBSchema.fromConnection(con, databaseName); - globalState.setSchema(newSchema); + while (globalState.getSchema().getDatabaseTables().size() < Randomly.getNotCachedInteger(1, 3)) { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = MariaDBTableGenerator.generate(tableName, globalState.getRandomly(), + globalState.getSchema()); + globalState.executeStatement(createTable); } int[] nrRemaining = new int[Action.values().length]; @@ -91,13 +77,16 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx case ANALYZE_TABLE: case UPDATE: case CREATE_INDEX: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getRandomly().getInteger(0, 2); + break; + case DELETE: + nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case SET: nrPerformed = 20; break; case INSERT: - nrPerformed = r.getInteger(0, options.getMaxNumberInserts()); + nrPerformed = globalState.getRandomly().getInteger(0, options.getMaxNumberInserts()); break; default: throw new AssertionError(action); @@ -110,7 +99,7 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx } while (total != 0) { Action nextAction = null; - int selection = r.getInteger(0, total); + int selection = globalState.getRandomly().getInteger(0, total); int previousRange = 0; for (int i = 0; i < nrRemaining.length; i++) { if (previousRange <= selection && selection < previousRange + nrRemaining[i]) { @@ -123,38 +112,41 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx assert nextAction != null; assert nrRemaining[nextAction.ordinal()] > 0; nrRemaining[nextAction.ordinal()]--; - Query query; + SQLQueryAdapter query; try { switch (nextAction) { case CHECKSUM: - query = MariaDBTableAdminCommandGenerator.checksumTable(newSchema); + query = MariaDBTableAdminCommandGenerator.checksumTable(globalState.getSchema()); break; case CHECK_TABLE: - query = MariaDBTableAdminCommandGenerator.checkTable(newSchema); + query = MariaDBTableAdminCommandGenerator.checkTable(globalState.getSchema()); break; case TRUNCATE: - query = MariaDBTruncateGenerator.truncate(newSchema); + query = MariaDBTruncateGenerator.truncate(globalState.getSchema()); break; case REPAIR_TABLE: - query = MariaDBTableAdminCommandGenerator.repairTable(newSchema); + query = MariaDBTableAdminCommandGenerator.repairTable(globalState.getSchema()); break; case INSERT: - query = MariaDBInsertGenerator.insert(newSchema, r); + query = MariaDBInsertGenerator.insert(globalState.getSchema(), globalState.getRandomly()); break; case OPTIMIZE: - query = MariaDBTableAdminCommandGenerator.optimizeTable(newSchema); + query = MariaDBTableAdminCommandGenerator.optimizeTable(globalState.getSchema()); break; case ANALYZE_TABLE: - query = MariaDBTableAdminCommandGenerator.analyzeTable(newSchema); + query = MariaDBTableAdminCommandGenerator.analyzeTable(globalState.getSchema()); break; case UPDATE: - query = MariaDBUpdateGenerator.update(newSchema, r); + query = MariaDBUpdateGenerator.update(globalState.getSchema(), globalState.getRandomly()); break; case CREATE_INDEX: - query = MariaDBIndexGenerator.generate(newSchema); + query = MariaDBIndexGenerator.generate(globalState.getSchema()); break; case SET: - query = MariaDBSetGenerator.set(r, options); + query = MariaDBSetGenerator.set(globalState.getRandomly(), options); + break; + case DELETE: + query = MariaDBDeleteGenerator.delete(globalState.getSchema(), globalState.getRandomly()); break; default: throw new AssertionError(nextAction); @@ -164,58 +156,41 @@ public void generateAndTestDatabase(MariaDBGlobalState globalState) throws SQLEx continue; } try { - if (options.logEachSelect()) { - logger.writeCurrent(query.getQueryString()); - } - manager.execute(query); - if (query.couldAffectSchema()) { - newSchema = MariaDBSchema.fromConnection(con, databaseName); - globalState.setSchema(newSchema); - } + globalState.executeStatement(query); } catch (Throwable t) { System.err.println(query.getQueryString()); throw t; } total--; } - newSchema = MariaDBSchema.fromConnection(con, databaseName); - // - MariaDBNoRECOracle queryGenerator = new MariaDBNoRECOracle(globalState); - for (int i = 0; i < options.getNrQueries(); i++) { - try { - queryGenerator.generateAndCheck(); - } catch (IgnoreMeException e) { - - } - manager.incrementSelectQueryCount(); - } - } - public static class MariaDBGlobalState extends GlobalState { - - private MariaDBSchema schema; - - public void setSchema(MariaDBSchema schema) { - this.schema = schema; - } + public static class MariaDBGlobalState extends SQLGlobalState { - public MariaDBSchema getSchema() { - return schema; + @Override + protected MariaDBSchema readSchema() throws SQLException { + return MariaDBSchema.fromConnection(getConnection(), getDatabaseName()); } } @Override - public Connection createDatabase(MariaDBGlobalState globalState) throws SQLException { - globalState.getState().statements - .add(new QueryAdapter("DROP DATABASE IF EXISTS " + globalState.getDatabaseName())); - globalState.getState().statements.add(new QueryAdapter("CREATE DATABASE " + globalState.getDatabaseName())); - globalState.getState().statements.add(new QueryAdapter("USE " + globalState.getDatabaseName())); - // /?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true - String url = "jdbc:mariadb://localhost:3306"; - Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); + public SQLConnection createDatabase(MariaDBGlobalState globalState) throws SQLException { + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + globalState.getDatabaseName()); + globalState.getState().logStatement("CREATE DATABASE " + globalState.getDatabaseName()); + globalState.getState().logStatement("USE " + globalState.getDatabaseName()); + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = MariaDBOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = MariaDBOptions.DEFAULT_PORT; + } + String url = String.format("jdbc:mariadb://%s:%d", host, port); + Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + globalState.getDatabaseName()); } @@ -225,7 +200,7 @@ public Connection createDatabase(MariaDBGlobalState globalState) throws SQLExcep try (Statement s = con.createStatement()) { s.execute("USE " + globalState.getDatabaseName()); } - return con; + return new SQLConnection(con); } @Override @@ -233,9 +208,4 @@ public String getDBMSName() { return "mariadb"; } - @Override - public String toString() { - return String.format("MariaDBProvider [database: %s]", databaseName); - } - } diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index afec7d6f9..7f7656d76 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -1,35 +1,36 @@ package sqlancer.mariadb; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.MariaDBSchema.MariaDBTable.MariaDBEngine; -public class MariaDBSchema { +public class MariaDBSchema extends AbstractSchema { private static final int NR_SCHEMA_READ_TRIES = 10; - private final List databaseTables; public enum MariaDBDataType { INT, VARCHAR, REAL, BOOLEAN; } - public static class MariaDBColumn implements Comparable { + public static class MariaDBColumn extends AbstractTableColumn { - private final String name; - private final MariaDBDataType columnType; private final boolean isPrimaryKey; - private MariaDBTable table; private final int precision; public enum CollateSequence { @@ -41,67 +42,22 @@ public static CollateSequence random() { } public MariaDBColumn(String name, MariaDBDataType columnType, boolean isPrimaryKey, int precision) { - this.name = name; - this.columnType = columnType; + super(name, null, columnType); this.isPrimaryKey = isPrimaryKey; this.precision = precision; } - @Override - public String toString() { - return String.format("%s.%s: %s", table.getName(), name, columnType); - } - - @Override - public int hashCode() { - return name.hashCode() + 11 * columnType.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof MariaDBColumn)) { - return false; - } else { - MariaDBColumn c = (MariaDBColumn) obj; - return table.getName().contentEquals(getName()) && name.equals(c.name); - } - } - - public String getName() { - return name; - } - - public String getFullQualifiedName() { - return table.getName() + "." + getName(); - } - - public MariaDBDataType getColumnType() { - return columnType; - } - public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } - public void setTable(MariaDBTable t) { - this.table = t; - } - - public MariaDBTable getTable() { - return table; - } - - @Override - public int compareTo(MariaDBColumn o) { - if (o.getTable().equals(this.getTable())) { - return name.compareTo(o.getName()); - } else { - return o.getTable().compareTo(table); - } + public static MariaDBColumn createDummy(String name) { + return new MariaDBColumn(name, MariaDBDataType.INT, false, 1); } } @@ -163,7 +119,7 @@ private static MariaDBDataType getColumnType(String typeString) { } } - public static class MariaDBTable implements Comparable { + public static class MariaDBTable extends AbstractRelationalTable { public enum MariaDBEngine { @@ -189,102 +145,38 @@ public static MariaDBEngine getRandomEngine() { } - private final String tableName; - private final List columns; - private final List indexes; private final MariaDBEngine engine; public MariaDBTable(String tableName, List columns, List indexes, MariaDBEngine engine) { - this.tableName = tableName; - this.indexes = indexes; + super(tableName, columns, indexes, false); this.engine = engine; - this.columns = Collections.unmodifiableList(columns); - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - sb.append(tableName + "\n"); - for (MariaDBColumn c : columns) { - sb.append("\t" + c + "\n"); - } - return sb.toString(); - } - - public List getIndexes() { - return indexes; - } - - public String getName() { - return tableName; - } - - public List getColumns() { - return columns; - } - - public String getColumnsAsString() { - return columns.stream().map(c -> c.getName()).collect(Collectors.joining(", ")); - } - - public String getColumnsAsString(Function function) { - return columns.stream().map(function).collect(Collectors.joining(", ")); - } - - public MariaDBColumn getRandomColumn() { - return Randomly.fromList(columns); - } - - public boolean hasIndexes() { - return !indexes.isEmpty(); - } - - public MariaDBIndex getRandomIndex() { - return Randomly.fromList(indexes); - } - - @Override - public int compareTo(MariaDBTable o) { - return o.getName().compareTo(tableName); - } - - public List getRandomNonEmptyColumnSubset() { - return Randomly.nonEmptySubset(getColumns()); } public MariaDBEngine getEngine() { return engine; } - public boolean hasPrimaryKey() { - return columns.stream().anyMatch(c -> c.isPrimaryKey()); - } } - public static final class MariaDBIndex { - - private final String indexName; + public static final class MariaDBIndex extends TableIndex { private MariaDBIndex(String indexName) { - this.indexName = indexName; - } - - public static MariaDBIndex create(String indexName) { - return new MariaDBIndex(indexName); + super(indexName); } + @Override public String getIndexName() { - if (indexName.contentEquals("PRIMARY")) { + if (super.getIndexName().contentEquals("PRIMARY")) { return "`PRIMARY`"; } else { - return indexName; + return super.getIndexName(); } } } - public static MariaDBSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static MariaDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { Exception ex = null; /* the loop is a workaround for https://bugs.MariaDB.com/bug.php?id=95929 */ for (int i = 0; i < NR_SCHEMA_READ_TRIES; i++) { @@ -316,7 +208,7 @@ public static MariaDBSchema fromConnection(Connection con, String databaseName) throw new AssertionError(ex); } - private static List getIndexes(Connection con, String tableName, String databaseName) + private static List getIndexes(SQLConnection con, String tableName, String databaseName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -325,14 +217,14 @@ private static List getIndexes(Connection con, String tableName, S databaseName, tableName))) { while (rs.next()) { String indexName = rs.getString("INDEX_NAME"); - indexes.add(MariaDBIndex.create(indexName)); + indexes.add(new MariaDBIndex(indexName)); } } } return indexes; } - private static List getTableColumns(Connection con, String tableName, String databaseName) + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -352,32 +244,11 @@ private static List getTableColumns(Connection con, String tableN } public MariaDBSchema(List databaseTables) { - this.databaseTables = Collections.unmodifiableList(databaseTables); - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - for (MariaDBTable t : getDatabaseTables()) { - sb.append(t + "\n"); - } - return sb.toString(); - } - - public MariaDBTable getRandomTable() { - return Randomly.fromList(getDatabaseTables()); + super(databaseTables); } public MariaDBTables getRandomTableNonEmptyTables() { - return new MariaDBTables(Randomly.nonEmptySubset(databaseTables)); - } - - public List getDatabaseTables() { - return databaseTables; - } - - public List getDatabaseTablesRandomSubsetNotEmpty() { - return Randomly.nonEmptySubset(databaseTables); + return new MariaDBTables(Randomly.nonEmptySubset(getDatabaseTables())); } } diff --git a/src/sqlancer/mariadb/ast/MariaDBAggregate.java b/src/sqlancer/mariadb/ast/MariaDBAggregate.java index 3977a6a95..ab5a47781 100644 --- a/src/sqlancer/mariadb/ast/MariaDBAggregate.java +++ b/src/sqlancer/mariadb/ast/MariaDBAggregate.java @@ -1,6 +1,6 @@ package sqlancer.mariadb.ast; -public class MariaDBAggregate extends MariaDBExpression { +public class MariaDBAggregate implements MariaDBExpression { private final MariaDBExpression expr; private final MariaDBAggregateFunction aggr; diff --git a/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java b/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java index 14f3c308a..5f56d168f 100644 --- a/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java +++ b/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBBinaryOperator extends MariaDBExpression { +public class MariaDBBinaryOperator implements MariaDBExpression { private MariaDBExpression left; private MariaDBExpression right; diff --git a/src/sqlancer/mariadb/ast/MariaDBColumnName.java b/src/sqlancer/mariadb/ast/MariaDBColumnName.java index a4fd78c1b..65d4cea0c 100644 --- a/src/sqlancer/mariadb/ast/MariaDBColumnName.java +++ b/src/sqlancer/mariadb/ast/MariaDBColumnName.java @@ -2,7 +2,7 @@ import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; -public class MariaDBColumnName extends MariaDBExpression { +public class MariaDBColumnName implements MariaDBExpression { private final MariaDBColumn column; diff --git a/src/sqlancer/mariadb/ast/MariaDBConstant.java b/src/sqlancer/mariadb/ast/MariaDBConstant.java index 499fe9ded..8670c9ed2 100644 --- a/src/sqlancer/mariadb/ast/MariaDBConstant.java +++ b/src/sqlancer/mariadb/ast/MariaDBConstant.java @@ -1,6 +1,9 @@ package sqlancer.mariadb.ast; -public class MariaDBConstant extends MariaDBExpression { +public class MariaDBConstant implements MariaDBExpression { + + private MariaDBConstant() { + } public static class MariaDBNullConstant extends MariaDBConstant { @@ -91,10 +94,6 @@ public static MariaDBTextConstant createTextConstant(String text) { return new MariaDBTextConstant(text); } - public static MariaDBDoubleConstant createDoubleConstant(double val) { - return new MariaDBDoubleConstant(val); - } - public static MariaDBIntConstant createIntConstant(long val) { return new MariaDBIntConstant(val); } diff --git a/src/sqlancer/mariadb/ast/MariaDBExpression.java b/src/sqlancer/mariadb/ast/MariaDBExpression.java index beff58866..d57c14888 100644 --- a/src/sqlancer/mariadb/ast/MariaDBExpression.java +++ b/src/sqlancer/mariadb/ast/MariaDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.mariadb.ast; -public class MariaDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; + +public interface MariaDBExpression extends Expression { } diff --git a/src/sqlancer/mariadb/ast/MariaDBFunction.java b/src/sqlancer/mariadb/ast/MariaDBFunction.java index fd033d310..481a26aaf 100644 --- a/src/sqlancer/mariadb/ast/MariaDBFunction.java +++ b/src/sqlancer/mariadb/ast/MariaDBFunction.java @@ -2,7 +2,7 @@ import java.util.List; -public class MariaDBFunction extends MariaDBExpression { +public class MariaDBFunction implements MariaDBExpression { private final MariaDBFunctionName func; private final List args; diff --git a/src/sqlancer/mariadb/ast/MariaDBInOperation.java b/src/sqlancer/mariadb/ast/MariaDBInOperation.java index 85981b390..15aad4daa 100644 --- a/src/sqlancer/mariadb/ast/MariaDBInOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBInOperation.java @@ -2,7 +2,7 @@ import java.util.List; -public class MariaDBInOperation extends MariaDBExpression { +public class MariaDBInOperation implements MariaDBExpression { private final MariaDBExpression expr; private final List list; diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java new file mode 100644 index 000000000..f1d0892e1 --- /dev/null +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -0,0 +1,86 @@ +package sqlancer.mariadb.ast; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; + +public class MariaDBJoin implements MariaDBExpression, Join { + + public enum JoinType { + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; + } + + private final MariaDBTable table; + private MariaDBExpression onClause; + private JoinType type; + + public MariaDBJoin(MariaDBJoin other) { + this.table = other.table; + this.onClause = other.onClause; + this.type = other.type; + } + + public MariaDBJoin(MariaDBTable table, MariaDBExpression onClause, JoinType type) { + this.table = table; + this.onClause = onClause; + this.type = type; + } + + public MariaDBTable getTable() { + return table; + } + + public MariaDBExpression getOnClause() { + return onClause; + } + + public JoinType getType() { + return type; + } + + @Override + public void setOnClause(MariaDBExpression onClause) { + this.onClause = onClause; + } + + public void setType(JoinType type) { + this.type = type; + } + + public static List getRandomJoinClauses(List tables, Randomly r) { + List joinStatements = new ArrayList<>(); + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + List columns = new ArrayList<>(); + if (tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + MariaDBTable table = Randomly.fromList(tables); + tables.remove(table); + columns.addAll(table.getColumns()); + MariaDBExpressionGenerator joinGen = new MariaDBExpressionGenerator(r).setColumns(columns); + MariaDBExpression joinClause = joinGen.getRandomExpression(); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + joinClause = null; + } + MariaDBJoin j = new MariaDBJoin(table, joinClause, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } +} diff --git a/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java b/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java index cd655a5d1..9b56a44dd 100644 --- a/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBPostfixUnaryOperation extends MariaDBExpression { +public class MariaDBPostfixUnaryOperation implements MariaDBExpression { private MariaDBPostfixUnaryOperator operator; private MariaDBExpression randomWhereCondition; diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index d1700f508..f09e7051f 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -3,37 +3,40 @@ import java.util.ArrayList; import java.util.List; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -public class MariaDBSelectStatement extends MariaDBExpression { +public class MariaDBSelectStatement extends SelectBase + implements MariaDBExpression, Select { public enum MariaDBSelectType { - ALL + ALL, DISTINCT, DISTINCTROW; } private List groupBys = new ArrayList<>(); private List columns = new ArrayList<>(); - private List tables = new ArrayList<>(); + private List joinClauses = new ArrayList<>(); private MariaDBSelectType selectType = MariaDBSelectType.ALL; private MariaDBExpression whereCondition; + @Override public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } + @Override public void setFetchColumns(List columns) { this.columns = columns; } - public void setFromTables(List tables) { - this.tables = tables; - } - public void setSelectType(MariaDBSelectType selectType) { this.selectType = selectType; } + @Override public void setWhereClause(MariaDBExpression whereCondition) { this.whereCondition = whereCondition; } @@ -50,12 +53,22 @@ public MariaDBSelectType getSelectType() { return selectType; } - public List getTables() { - return tables; - } - public MariaDBExpression getWhereCondition() { return whereCondition; } + @Override + public List getJoinClauses() { + return joinClauses; + } + + @Override + public void setJoinClauses(List joinClauses) { + this.joinClauses = joinClauses; + } + + @Override + public String asString() { + return MariaDBVisitor.asString(this); + } } diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index 53ed490d0..f7fed0c83 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.mariadb.ast; import java.util.List; -import java.util.stream.Collectors; public class MariaDBStringVisitor extends MariaDBVisitor { @@ -27,7 +26,7 @@ public void visit(MariaDBPostfixUnaryOperation op) { @Override public void visit(MariaDBColumnName c) { - sb.append(c.getColumn().getName()); + sb.append(c.getColumn().getFullQualifiedName()); } @Override @@ -41,13 +40,22 @@ public void visit(MariaDBSelectStatement s) { visit(column); } sb.append(" FROM "); - sb.append(s.getTables().stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + + for (int j = 0; j < s.getFromList().size(); j++) { + if (j != 0) { + sb.append(", "); + } + visit(s.getFromList().get(j)); + } + for (MariaDBExpression j : s.getJoinClauses()) { + visit(j); + } if (s.getWhereCondition() != null) { sb.append(" WHERE "); visit(s.getWhereCondition()); } - if (s.getGroupBys().size() != 0) { - sb.append(" GROUP BY"); + if (!s.getGroupBys().isEmpty()) { + sb.append(" GROUP BY "); for (i = 0; i < s.getGroupBys().size(); i++) { if (i != 0) { sb.append(", "); @@ -131,4 +139,41 @@ private void visitList(List list) { } } + @Override + public void visit(MariaDBJoin join) { + sb.append(" "); + switch (join.getType()) { + case NATURAL: + sb.append("NATURAL "); + break; + case INNER: + sb.append("INNER "); + break; + case STRAIGHT: + sb.append("STRAIGHT_"); + break; + case LEFT: + sb.append("LEFT "); + break; + case RIGHT: + sb.append("RIGHT "); + break; + case CROSS: + sb.append("CROSS "); + break; + default: + throw new AssertionError(join.getType()); + } + sb.append("JOIN "); + sb.append(join.getTable().getName()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + @Override + public void visit(MariaDBTableReference ref) { + sb.append(ref.getTable().getName()); + } } diff --git a/src/sqlancer/mariadb/ast/MariaDBTableReference.java b/src/sqlancer/mariadb/ast/MariaDBTableReference.java new file mode 100644 index 000000000..f045907ca --- /dev/null +++ b/src/sqlancer/mariadb/ast/MariaDBTableReference.java @@ -0,0 +1,16 @@ +package sqlancer.mariadb.ast; + +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; + +public class MariaDBTableReference implements MariaDBExpression { + + private final MariaDBTable table; + + public MariaDBTableReference(MariaDBTable table) { + this.table = table; + } + + public MariaDBTable getTable() { + return table; + } +} diff --git a/src/sqlancer/mariadb/ast/MariaDBText.java b/src/sqlancer/mariadb/ast/MariaDBText.java index b96871063..c3d1c2d1a 100644 --- a/src/sqlancer/mariadb/ast/MariaDBText.java +++ b/src/sqlancer/mariadb/ast/MariaDBText.java @@ -1,6 +1,6 @@ package sqlancer.mariadb.ast; -public class MariaDBText extends MariaDBExpression { +public class MariaDBText implements MariaDBExpression { private final MariaDBExpression expr; private final String text; diff --git a/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java b/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java index af229ce6e..dda61dbee 100644 --- a/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBUnaryPrefixOperation extends MariaDBExpression { +public class MariaDBUnaryPrefixOperation implements MariaDBExpression { private MariaDBExpression expr; private MariaDBUnaryPrefixOperator op; diff --git a/src/sqlancer/mariadb/ast/MariaDBVisitor.java b/src/sqlancer/mariadb/ast/MariaDBVisitor.java index 8ea3f9d37..8626dc967 100644 --- a/src/sqlancer/mariadb/ast/MariaDBVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBVisitor.java @@ -22,6 +22,10 @@ public abstract class MariaDBVisitor { public abstract void visit(MariaDBInOperation op); + public abstract void visit(MariaDBJoin join); + + public abstract void visit(MariaDBTableReference join); + public void visit(MariaDBExpression expr) { if (expr instanceof MariaDBConstant) { visit((MariaDBConstant) expr); @@ -43,6 +47,10 @@ public void visit(MariaDBExpression expr) { visit((MariaDBFunction) expr); } else if (expr instanceof MariaDBInOperation) { visit((MariaDBInOperation) expr); + } else if (expr instanceof MariaDBJoin) { + visit((MariaDBJoin) expr); + } else if (expr instanceof MariaDBTableReference) { + visit((MariaDBTableReference) expr); } else { throw new AssertionError(expr.getClass()); } diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java new file mode 100644 index 000000000..da1651662 --- /dev/null +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -0,0 +1,99 @@ +package sqlancer.mariadb.gen; + +import java.util.Collections; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTables; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.ast.MariaDBVisitor; + +public final class MariaDBDeleteGenerator extends AbstractDeleteGenerator { + + private final MariaDBSchema schema; + private final Randomly r; + + private MariaDBDeleteGenerator(MariaDBSchema schema, Randomly r) { + this.schema = schema; + this.r = r; + } + + public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { + return new MariaDBDeleteGenerator(schema, r).getStatement(); + } + + @Override + public void buildStatement() { + MariaDBTable table = schema.getRandomTable(); + + MariaDBExpressionGenerator expressionGenerator = new MariaDBExpressionGenerator(r); + + AbstractTables tablesAndColumns = new AbstractTables<>( + Collections.singletonList(table)); + expressionGenerator.setTablesAndColumns(tablesAndColumns); + + errors.add("foreign key constraint fails"); + errors.add("cannot delete or update a parent row"); + errors.add("Data truncated"); + errors.add("Division by 0"); + errors.add("Incorrect value"); + + sb.append("DELETE"); + + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" LOW_PRIORITY"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" QUICK"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" IGNORE"); + } + + sb.append(" FROM "); + sb.append(table.getName()); + + if (Randomly.getBoolean()) { + String condition; + if (Randomly.getBooleanWithRatherLowProbability()) { + condition = MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r)); + } else { + condition = MariaDBVisitor.asString(expressionGenerator.getRandomExpression()); + } + appendWhereClause(condition); + } + + // ORDER BY + LIMIT + if (Randomly.getBooleanWithRatherLowProbability() && !table.getColumns().isEmpty()) { + sb.append(" ORDER BY "); + sb.append(Randomly.fromList(table.getColumns()).getName()); + if (Randomly.getBoolean()) { + sb.append(Randomly.getBoolean() ? " ASC" : " DESC"); + } + } + + if (Randomly.getBooleanWithRatherLowProbability()) { + appendLimitClause(Randomly.getNotCachedInteger(1, 10)); + } + + // RETURNING clause (MariaDB >= 10.5) + if (Randomly.getBooleanWithRatherLowProbability()) { + String expression; + if (Randomly.getBooleanWithRatherLowProbability()) { + expression = MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r)); + } else { + expression = MariaDBVisitor.asString(expressionGenerator.getRandomExpression()); + } + appendReturningClause(expression); + } + + if (sb.toString().contains("RLIKE") || sb.toString().contains("REGEXP")) { + errors.add("Regex error"); + errors.add("quantifier does not follow a repeatable item"); + errors.add("Got error"); + } + } +} diff --git a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java index c15c21b06..3b7386121 100644 --- a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java @@ -1,15 +1,18 @@ package sqlancer.mariadb.gen; -import java.sql.Connection; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import sqlancer.Randomly; -import sqlancer.StateToReproduce; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.mariadb.MariaDBProvider; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.ast.MariaDBAggregate; +import sqlancer.mariadb.ast.MariaDBAggregate.MariaDBAggregateFunction; import sqlancer.mariadb.ast.MariaDBBinaryOperator; import sqlancer.mariadb.ast.MariaDBBinaryOperator.MariaDBBinaryComparisonOperator; import sqlancer.mariadb.ast.MariaDBColumnName; @@ -18,14 +21,21 @@ import sqlancer.mariadb.ast.MariaDBFunction; import sqlancer.mariadb.ast.MariaDBFunctionName; import sqlancer.mariadb.ast.MariaDBInOperation; +import sqlancer.mariadb.ast.MariaDBJoin; import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation; import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation.MariaDBPostfixUnaryOperator; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBSelectStatement.MariaDBSelectType; +import sqlancer.mariadb.ast.MariaDBTableReference; +import sqlancer.mariadb.ast.MariaDBText; import sqlancer.mariadb.ast.MariaDBUnaryPrefixOperation; import sqlancer.mariadb.ast.MariaDBUnaryPrefixOperation.MariaDBUnaryPrefixOperator; -public class MariaDBExpressionGenerator { +public class MariaDBExpressionGenerator + implements NoRECGenerator { private final Randomly r; + private List targetTables = new ArrayList<>(); private List columns = new ArrayList<>(); public MariaDBExpressionGenerator(Randomly r) { @@ -66,14 +76,6 @@ public MariaDBExpressionGenerator setColumns(List columns) { return this; } - public MariaDBExpressionGenerator setCon(Connection con) { - return this; - } - - public MariaDBExpressionGenerator setState(StateToReproduce state) { - return this; - } - private enum ExpressionType { LITERAL, COLUMN, BINARY_COMPARISON, UNARY_POSTFIX_OPERATOR, UNARY_PREFIX_OPERATOR, FUNCTION, IN } @@ -146,4 +148,64 @@ public MariaDBExpression getRandomExpression() { return getRandomExpression(0); } + @Override + public MariaDBExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + this.targetTables = targetTables.getTables(); + this.columns = targetTables.getColumns(); + return this; + } + + @Override + public List getTableRefs() { + List tableRefs = new ArrayList<>(); + for (MariaDBTable t : targetTables) { + MariaDBTableReference tableRef = new MariaDBTableReference(t); + tableRefs.add(tableRef); + } + return tableRefs; + } + + @Override + public MariaDBExpression generateBooleanExpression() { + return getRandomExpression(); + } + + @Override + public MariaDBSelectStatement generateSelect() { + return new MariaDBSelectStatement(); + } + + @Override + public List getRandomJoinClauses() { + return MariaDBJoin.getRandomJoinClauses(targetTables, r); + } + + @Override + public String generateOptimizedQueryString(MariaDBSelectStatement select, MariaDBExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + MariaDBAggregate aggr = new MariaDBAggregate( + new MariaDBColumnName(new MariaDBColumn("*", MariaDBDataType.INT, false, 0)), + MariaDBAggregateFunction.COUNT); + select.setFetchColumns(Arrays.asList(aggr)); + } else { + MariaDBColumnName aggr = new MariaDBColumnName(MariaDBColumn.createDummy("*")); + select.setFetchColumns(Arrays.asList(aggr)); + } + + select.setWhereClause(whereCondition); + select.setSelectType(MariaDBSelectType.ALL); + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(MariaDBSelectStatement select, MariaDBExpression whereCondition) { + MariaDBPostfixUnaryOperation isTrue = new MariaDBPostfixUnaryOperation(MariaDBPostfixUnaryOperator.IS_TRUE, + whereCondition); + MariaDBText asText = new MariaDBText(isTrue, " as count", false); + select.setFetchColumns(Arrays.asList(asText)); + select.setSelectType(MariaDBSelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as asdf"; + } } diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 39579e358..1fb6f10b3 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -1,60 +1,47 @@ package sqlancer.mariadb.gen; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -import sqlancer.sqlite3.gen.SQLite3Common; -public final class MariaDBIndexGenerator { +public class MariaDBIndexGenerator extends AbstractIndexGenerator { + + private final MariaDBSchema schema; + + public MariaDBIndexGenerator(MariaDBSchema schema) { + this.schema = schema; + this.canAffectSchema = true; + } - private MariaDBIndexGenerator() { + public static SQLQueryAdapter generate(MariaDBSchema s) { + return new MariaDBIndexGenerator(s).getStatement(); } - public static Query generate(MariaDBSchema s) { - List errors = new ArrayList<>(); - StringBuilder sb = new StringBuilder("CREATE "); + @Override + public void buildStatement() { errors.add("Key/Index cannot be defined on a virtual generated column"); - if (Randomly.getBoolean()) { + errors.add("Specified key was too long"); + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Duplicate entry"); errors.add("Key/Index cannot be defined on a virtual generated column"); - sb.append("UNIQUE "); } - sb.append("INDEX "); - sb.append("i" + SQLite3Common.createColumnName(Randomly.smallNumber())); + appendCreateIndex(unique); + sb.append("i"); + sb.append(DBMSCommon.createColumnName(Randomly.smallNumber())); if (Randomly.getBoolean()) { sb.append(" USING "); sb.append(Randomly.fromOptions("BTREE", "HASH")); // , "RTREE") } sb.append(" ON "); - MariaDBTable randomTable = s.getRandomTable(); + MariaDBTable randomTable = schema.getRandomTable(); sb.append(randomTable.getName()); - sb.append("("); - List columns = Randomly.nonEmptySubset(randomTable.getColumns()); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("ASC", "DESC")); - } - } - sb.append(")"); - // if (Randomly.getBoolean()) { - // sb.append(" ALGORITHM="); - // sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY", "NOCOPY", "INSTANT")); - // errors.add("is not supported for this operation"); - // } - - return new QueryAdapter(sb.toString(), errors, true); + appendIndexColumnList(Randomly.nonEmptySubset(randomTable.getColumns()), true); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java index b27e127c1..5fa906727 100644 --- a/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBInsertGenerator.java @@ -1,11 +1,8 @@ package sqlancer.mariadb.gen; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -16,7 +13,7 @@ public final class MariaDBInsertGenerator { private MariaDBInsertGenerator() { } - public static Query insert(MariaDBSchema s, Randomly r) { + public static SQLQueryAdapter insert(MariaDBSchema s, Randomly r) { MariaDBTable randomTable = s.getRandomTable(); StringBuilder sb = new StringBuilder(); sb.append("INSERT INTO "); @@ -29,14 +26,14 @@ public static Query insert(MariaDBSchema s, Randomly r) { if (Randomly.getBooleanWithSmallProbability()) { sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); } else { - sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r, - randomTable.getColumns().get(i).getColumnType()))); + sb.append(MariaDBVisitor.asString( + MariaDBExpressionGenerator.getRandomConstant(r, randomTable.getColumns().get(i).getType()))); } } sb.append(")"); - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); MariaDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index 270862d76..860f60c4f 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -1,14 +1,17 @@ package sqlancer.mariadb.gen; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.MainOptions; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBBugs; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; public class MariaDBSetGenerator { @@ -23,7 +26,7 @@ public MariaDBSetGenerator(Randomly r, MainOptions options) { this.isSingleThreaded = options.getNumberConcurrentThreads() == 1; } - public static Query set(Randomly r, MainOptions options) { + public static SQLQueryAdapter set(Randomly r, MainOptions options) { return new MariaDBSetGenerator(r, options).get(); } @@ -35,13 +38,12 @@ private enum Action { AUTOCOMMIT("autocommit", (r) -> 1, Scope.GLOBAL, Scope.SESSION), // BIG_TABLES("big_tables", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - COMPLETION_TYPE("completion_type", (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", 0, 1, 2), - Scope.GLOBAL), // + COMPLETION_TYPE("completion_type", + (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", "0", "1", "2"), Scope.GLOBAL), // // BULK_INSERT_CACHE_SIZE("bulk_insert_buffer_size", (r) -> r.getLong(0, Long.MAX_VALUE), Scope.GLOBAL, // Scope.SESSION), - CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", 0, 1, 2), + CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", "0", "1", "2"), Scope.GLOBAL), - CTE_MAX_RECURSION_DEPTH("cte_max_recursion_depth", (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), DELAY_KEY_WRITE("delay_key_write", (r) -> Randomly.fromOptions("ON", "OFF", "ALL"), Scope.GLOBAL), EQ_RANGE_INDEX_DIVE_LIMIT("eq_range_index_dive_limit", (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), FLUSH("flush", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL), @@ -86,7 +88,6 @@ private enum Action { */ // READ_BUFFER_SIZE("read_buffer_size", (r) -> r.getLong(8200, 2147479552), Scope.GLOBAL, Scope.SESSION), // READ_RND_BUFFER_SIZE("read_rnd_buffer_size", (r) -> r.getLong(1, 2147483647), Scope.GLOBAL, Scope.SESSION), - SCHEMA_DEFINITION_CACHE("schema_definition_cache", (r) -> r.getLong(256, 524288), Scope.GLOBAL), /* * sort_buffer_size is commented out as a workaround for https://bugs.mysql.com/bug.php?id=95969 */ @@ -120,27 +121,15 @@ private enum Action { private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); - String[] options = { /* - * ("batched_key_access", /*"block_nested_loop", "condition_fanout_filter", - */ - "condition_pushdown_for_derived", // MariaDB - "derived_merge", // - "derived_with_keys", // MariaDB - "engine_condition_pushdown", // - "exists_to_in", // MariaDB - "extended_keys", // MariaDB - "firstmatch", // MariaDB - "index_condition_pushdown", // - /* "use_index_extensions", */ - "index_merge", // - "index_merge_intersection", // - "index_merge_sort_intersection", // - "index_merge_sort_union", // - "index_merge_union", "in_to_exists", // MariaDB - /* "use_invisible_indexes", */ "mrr", "mrr_cost_based", /* "skip_scan", */ "semijoin", /* - * "duplicateweedout", - */ - "firstmatch", "loosescan", "materialization", /* "subquery_materialization_cost_based" */ }; + String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "exists_to_in", + "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", "index_merge", + "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", + "index_merge_union", "in_to_exists", "join_cache_bka", "join_cache_hashed", + "join_cache_incremental", "loosescan", "materialization", "mrr", "mrr_cost_based", "mrr_sort_keys", + "not_null_range_scan", "optimize_join_buffer_size", "orderby_uses_equalities", + "outer_join_with_cache", "partial_match_rowid_merge", "partial_match_table_scan", "rowid_filter", + "semijoin", "semijoin_with_cache", "split_materialized", "subquery_cache", "table_elimination" }; List optionSubset = Arrays.asList(Randomly.fromOptions(options)); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); @@ -162,7 +151,7 @@ public Scope[] getScopes() { } } - private Query get() { + private SQLQueryAdapter get() { sb.append("SET "); Action a; if (isSingleThreaded) { @@ -190,7 +179,64 @@ private Query get() { sb.append(a.name); sb.append(" = "); sb.append(a.prod.apply(r)); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors + .from("At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'")); + } + + public static SQLQueryAdapter resetOptimizer() { + return new SQLQueryAdapter("SET optimizer_switch='default'"); + } + + public static List getAllOptimizer(MariaDBGlobalState globalState) { + List result = new ArrayList<>(); + String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "exists_to_in", "extended_keys", + "firstmatch", "index_condition_pushdown", "hash_join_cardinality", "index_merge", + "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", + "index_merge_union", "in_to_exists", "join_cache_bka", "join_cache_hashed", "join_cache_incremental", + "loosescan", "materialization", "mrr", "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", + "optimize_join_buffer_size", "orderby_uses_equalities", "outer_join_with_cache", + "partial_match_rowid_merge", "partial_match_table_scan", "rowid_filter", "semijoin", + "semijoin_with_cache", "split_materialized", "subquery_cache", "table_elimination" }; + List availableOptions = new ArrayList<>(Arrays.asList(options)); + if (MariaDBBugs.bug21058) { + availableOptions.remove("in_to_exists"); // https://jira.mariadb.org/browse/MDEV-21058 + } + if (MariaDBBugs.bug32076) { + availableOptions.remove("not_null_range_scan"); // https://jira.mariadb.org/browse/MDEV-32076 + } + if (MariaDBBugs.bug32099) { + availableOptions.remove("optimize_join_buffer_size"); // https://jira.mariadb.org/browse/MDEV-32099 + } + if (MariaDBBugs.bug32105) { + availableOptions.remove("join_cache_hashed"); // https://jira.mariadb.org/browse/MDEV-32105 + } + if (MariaDBBugs.bug32106) { + availableOptions.remove("outer_join_with_cache"); // https://jira.mariadb.org/browse/MDEV-32106 + } + if (MariaDBBugs.bug32107) { + availableOptions.remove("table_elimination"); // https://jira.mariadb.org/browse/MDEV-32107 + } + if (MariaDBBugs.bug32108) { + availableOptions.remove("join_cache_incremental"); // https://jira.mariadb.org/browse/MDEV-32108 + } + if (MariaDBBugs.bug32143) { + availableOptions.remove("mrr"); // https://jira.mariadb.org/browse/MDEV-32143 + } + + StringBuilder sb = new StringBuilder(); + sb.append("SET SESSION optimizer_switch = '%s'"); + + for (String option : availableOptions) { + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=on"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=off"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=default"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + } + + return result; } } diff --git a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java index 62b2d3640..dd40739f4 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java @@ -5,10 +5,10 @@ import java.util.function.Function; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryResultCheckAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryResultCheckAdapter; +import sqlancer.mariadb.MariaDBBugs; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -17,16 +17,16 @@ public final class MariaDBTableAdminCommandGenerator { private MariaDBTableAdminCommandGenerator() { } - public static Query checksumTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter checksumTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "CHECKSUM TABLE"); if (Randomly.getBoolean()) { sb.append(" "); sb.append(Randomly.fromOptions("QUICK", "EXTENDED")); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } - public static Query repairTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter repairTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "REPAIR TABLE"); if (Randomly.getBoolean()) { List subset = Randomly.nonEmptySubset("QUICK", "EXTENDED"); // , "USE_FRM" @@ -37,12 +37,12 @@ public static Query repairTable(MariaDBSchema newSchema) { s -> s.equals("OK") || s.equals("The storage engine for the table doesn't support repair")); } - public static Query analyzeTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter analyzeTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "ANALYZE TABLE"); return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table is already up to date")); } - public static Query checkTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter checkTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "CHECK TABLE"); if (Randomly.getBoolean()) { List subset = Randomly.nonEmptySubset("FOR UPGRADE", "QUICK", "FAST", "MEDIUM", "EXTENDED", @@ -53,16 +53,19 @@ public static Query checkTable(MariaDBSchema newSchema) { return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table is already up to date")); } - public static Query optimizeTable(MariaDBSchema newSchema) { + public static SQLQueryAdapter optimizeTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "OPTIMIZE TABLE"); - MariaDBCommon.addWaitClause(sb); + if (!MariaDBBugs.bug33893) { + MariaDBCommon.addWaitClause(sb); + } return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table does not support optimize, doing recreate + analyze instead") - || s.contentEquals("Table is already up to date")); + || s.contentEquals("Table is already up to date") || s.contains("Lock wait timeout") + || s.contains("Operation failed")); } - private static Query checkForMsgText(StringBuilder sb, Function checker) { - return new QueryResultCheckAdapter(sb.toString(), rs -> { + private static SQLQueryAdapter checkForMsgText(StringBuilder sb, Function checker) { + return new SQLQueryResultCheckAdapter(sb.toString(), rs -> { try { while (rs.next()) { String s = rs.getString("Msg_text"); diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index 9c6690074..23f09a420 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -4,15 +4,15 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBBugs; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; import sqlancer.mariadb.MariaDBSchema.MariaDBTable.MariaDBEngine; import sqlancer.mariadb.ast.MariaDBVisitor; -import sqlancer.sqlite3.gen.SQLite3Common; public class MariaDBTableGenerator { @@ -22,7 +22,7 @@ public class MariaDBTableGenerator { private PrimaryKeyState primaryKeyState = Randomly.fromOptions(PrimaryKeyState.values()); private final List columnNames = new ArrayList<>(); private final Randomly r; - private final List errors = new ArrayList<>(); + private final ExpectedErrors errors = new ExpectedErrors(); public MariaDBTableGenerator(String tableName, Randomly r, MariaDBSchema newSchema) { this.tableName = tableName; @@ -30,17 +30,17 @@ public MariaDBTableGenerator(String tableName, Randomly r, MariaDBSchema newSche this.r = r; } - public static Query generate(String tableName, Randomly r, MariaDBSchema newSchema) { + public static SQLQueryAdapter generate(String tableName, Randomly r, MariaDBSchema newSchema) { return new MariaDBTableGenerator(tableName, r, newSchema).gen(); } - private Query gen() { + private SQLQueryAdapter gen() { if (Randomly.getBoolean() || s.getDatabaseTables().isEmpty()) { newTable(); } else { likeOtherTable(); } - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors, true); } private enum PrimaryKeyState { @@ -54,7 +54,7 @@ private void newTable() { if (i != 0) { sb.append(", "); } - String columnName = SQLite3Common.createColumnName(i); + String columnName = DBMSCommon.createColumnName(i); columnNames.add(columnName); sb.append(columnName); sb.append(" "); @@ -142,6 +142,7 @@ private void createOrReplaceTable() { sb.append("IF NOT EXISTS "); } sb.append(tableName); + errors.add("Specified key was too long; max key length is"); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java index 2b4da154c..ecc240bb3 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java @@ -1,7 +1,8 @@ package sqlancer.mariadb.gen; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; public final class MariaDBTruncateGenerator { @@ -9,12 +10,14 @@ public final class MariaDBTruncateGenerator { private MariaDBTruncateGenerator() { } - public static Query truncate(MariaDBSchema s) { + public static SQLQueryAdapter truncate(MariaDBSchema s) { StringBuilder sb = new StringBuilder("TRUNCATE "); sb.append(s.getRandomTable().getName()); sb.append(" "); MariaDBCommon.addWaitClause(sb); - return new QueryAdapter(sb.toString()); + ExpectedErrors errors = new ExpectedErrors(); + MariaDBErrors.addCommonErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java index 3e2e0f9ee..8a9127961 100644 --- a/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBUpdateGenerator.java @@ -1,11 +1,8 @@ package sqlancer.mariadb.gen; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -16,7 +13,7 @@ public final class MariaDBUpdateGenerator { private MariaDBUpdateGenerator() { } - public static Query update(MariaDBSchema s, Randomly r) { + public static SQLQueryAdapter update(MariaDBSchema s, Randomly r) { MariaDBTable randomTable = s.getRandomTable(); StringBuilder sb = new StringBuilder("UPDATE "); if (Randomly.getBoolean()) { @@ -40,9 +37,9 @@ public static Query update(MariaDBSchema s, Randomly r) { } // [WHERE where_condition] [ORDER BY ...] [LIMIT row_count] } - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); MariaDBErrors.addInsertErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java new file mode 100644 index 000000000..dcd458193 --- /dev/null +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -0,0 +1,92 @@ +package sqlancer.mariadb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBErrors; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.MariaDBSchema.MariaDBTables; +import sqlancer.mariadb.ast.MariaDBColumnName; +import sqlancer.mariadb.ast.MariaDBExpression; +import sqlancer.mariadb.ast.MariaDBJoin; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBTableReference; +import sqlancer.mariadb.ast.MariaDBVisitor; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; +import sqlancer.mariadb.gen.MariaDBSetGenerator; + +public class MariaDBDQPOracle implements TestOracle { + private final MariaDBGlobalState state; + private final MariaDBSchema s; + private MariaDBExpressionGenerator gen; + private MariaDBSelectStatement select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public MariaDBDQPOracle(MariaDBGlobalState globalState) { + state = globalState; + s = globalState.getSchema(); + MariaDBErrors.addCommonErrors(errors); + } + + @Override + public void check() throws Exception { + MariaDBTables tables = s.getRandomTableNonEmptyTables(); + gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(tables.getColumns()); + + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new MariaDBColumnName(c)) + .collect(Collectors.toList())); + + select = new MariaDBSelectStatement(); + select.setFetchColumns(fetchColumns); + + select.setSelectType(Randomly.fromOptions(MariaDBSelectStatement.MariaDBSelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.getRandomExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByClause(fetchColumns); + } + + // Set the join. + List joinExpressions = MariaDBJoin.getRandomJoinClauses(tables.getTables(), state.getRandomly()); + select.setJoinClauses(joinExpressions); + + // Set the from clause from the tables that are not used in the join. + select.setFromList( + tables.getTables().stream().map(t -> new MariaDBTableReference(t)).collect(Collectors.toList())); + + // Get the result of the first query + String originalQueryString = MariaDBVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + List optimizationList = MariaDBSetGenerator.getAllOptimizer(state); + for (SQLQueryAdapter optimization : optimizationList) { + optimization.execute(state); + List result = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, + List.of(originalQueryString), state); + } catch (AssertionError e) { + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + originalResult.size(), result.size(), originalQueryString, originalResult.size(), + String.join(";", originalQueryString), result.size()); + assertionMessage += System.lineSeparator() + "The setting: " + optimization.getQueryString(); + throw new AssertionError(assertionMessage); + } + } + + } + +} diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java deleted file mode 100644 index 8e4a28b78..000000000 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ /dev/null @@ -1,139 +0,0 @@ -package sqlancer.mariadb.oracle; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; -import sqlancer.Randomly; -import sqlancer.StateToReproduce; -import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.MariaDBSchema; -import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; -import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; -import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -import sqlancer.mariadb.ast.MariaDBAggregate; -import sqlancer.mariadb.ast.MariaDBAggregate.MariaDBAggregateFunction; -import sqlancer.mariadb.ast.MariaDBColumnName; -import sqlancer.mariadb.ast.MariaDBExpression; -import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation; -import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation.MariaDBPostfixUnaryOperator; -import sqlancer.mariadb.ast.MariaDBSelectStatement; -import sqlancer.mariadb.ast.MariaDBSelectStatement.MariaDBSelectType; -import sqlancer.mariadb.ast.MariaDBText; -import sqlancer.mariadb.ast.MariaDBVisitor; -import sqlancer.mariadb.gen.MariaDBExpressionGenerator; - -public class MariaDBNoRECOracle { - - private final MariaDBSchema s; - private final Randomly r; - private final Connection con; - private String firstQueryString; - private String secondQueryString; - private final List errors = new ArrayList<>(); - private static final int NOT_FOUND = -1; - private final StateToReproduce state; - private final MariaDBGlobalState globalState; - - public MariaDBNoRECOracle(MariaDBGlobalState globalState) { - this.s = globalState.getSchema(); - this.r = globalState.getRandomly(); - this.con = globalState.getConnection(); - this.state = globalState.getState(); - this.globalState = globalState; - errors.add("is out of range"); - // regex - errors.add("unmatched parentheses"); - errors.add("nothing to repeat at offset"); - errors.add("missing )"); - errors.add("missing terminating ]"); - errors.add("range out of order in character class"); - errors.add("unrecognized character after "); - errors.add("Got error '(*VERB) not recognized or malformed"); - errors.add("must be followed by"); - errors.add("malformed number or name after"); - errors.add("digit expected after"); - } - - public void generateAndCheck() throws SQLException { - MariaDBTable randomTable = s.getRandomTable(); - List columns = randomTable.getColumns(); - MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(r).setColumns(columns).setCon(con) - .setState(state); - MariaDBExpression randomWhereCondition = gen.getRandomExpression(); - List groupBys = Collections.emptyList(); // getRandomExpressions(columns); - int optimizedCount = getOptimizedQuery(randomTable, randomWhereCondition, groupBys); - int unoptimizedCount = getUnoptimizedQuery(randomTable, randomWhereCondition, groupBys); - if (optimizedCount == NOT_FOUND || unoptimizedCount == NOT_FOUND) { - throw new IgnoreMeException(); - } - if (optimizedCount != unoptimizedCount) { - state.queryString = firstQueryString + ";\n" + secondQueryString + ";"; - throw new AssertionError(optimizedCount + " " + unoptimizedCount); - } - } - - private int getUnoptimizedQuery(MariaDBTable randomTable, MariaDBExpression randomWhereCondition, - List groupBys) throws SQLException { - MariaDBSelectStatement select = new MariaDBSelectStatement(); - select.setGroupByClause(groupBys); - MariaDBPostfixUnaryOperation isTrue = new MariaDBPostfixUnaryOperation(MariaDBPostfixUnaryOperator.IS_TRUE, - randomWhereCondition); - MariaDBText asText = new MariaDBText(isTrue, " as count", false); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromTables(Arrays.asList(randomTable)); - select.setSelectType(MariaDBSelectType.ALL); - int secondCount = 0; - - secondQueryString = "SELECT SUM(count) FROM (" + MariaDBVisitor.asString(select) + ") as asdf"; - QueryAdapter q = new QueryAdapter(secondQueryString, errors); - try (ResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return NOT_FOUND; - } else { - while (rs.next()) { - secondCount = rs.getInt(1); - rs.getStatement().close(); - } - rs.getStatement().close(); - } - } - - return secondCount; - } - - private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression randomWhereCondition, - List groupBys) throws SQLException { - MariaDBSelectStatement select = new MariaDBSelectStatement(); - select.setGroupByClause(groupBys); - MariaDBAggregate aggr = new MariaDBAggregate( - new MariaDBColumnName(new MariaDBColumn("*", MariaDBDataType.INT, false, 0)), - MariaDBAggregateFunction.COUNT); - select.setFetchColumns(Arrays.asList(aggr)); - select.setFromTables(Arrays.asList(randomTable)); - select.setWhereClause(randomWhereCondition); - select.setSelectType(MariaDBSelectType.ALL); - int firstCount = 0; - firstQueryString = MariaDBVisitor.asString(select); - QueryAdapter q = new QueryAdapter(firstQueryString, errors); - try (ResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - firstCount = NOT_FOUND; - } else { - rs.next(); - firstCount = rs.getInt(1); - rs.getStatement().close(); - } - } catch (Exception e) { - throw new AssertionError(firstQueryString, e); - } - return firstCount; - } - -} diff --git a/src/sqlancer/materialize/MaterializeBugs.java b/src/sqlancer/materialize/MaterializeBugs.java new file mode 100644 index 000000000..a7611a925 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeBugs.java @@ -0,0 +1,12 @@ +package sqlancer.materialize; + +// do not make the fields final to avoid warnings +public final class MaterializeBugs { + + // Tables or columns may be missing when reading information_schema shortly after creation + public static boolean bugSchemaReadIncomplete = true; + + private MaterializeBugs() { + } + +} diff --git a/src/sqlancer/materialize/MaterializeCompoundDataType.java b/src/sqlancer/materialize/MaterializeCompoundDataType.java new file mode 100644 index 000000000..a4bddf7ca --- /dev/null +++ b/src/sqlancer/materialize/MaterializeCompoundDataType.java @@ -0,0 +1,46 @@ +package sqlancer.materialize; + +import java.util.Optional; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public final class MaterializeCompoundDataType { + + private final MaterializeDataType dataType; + private final MaterializeCompoundDataType elemType; + private final Integer size; + + private MaterializeCompoundDataType(MaterializeDataType dataType, MaterializeCompoundDataType elemType, + Integer size) { + this.dataType = dataType; + this.elemType = elemType; + this.size = size; + } + + public MaterializeDataType getDataType() { + return dataType; + } + + public MaterializeCompoundDataType getElemType() { + if (elemType == null) { + throw new AssertionError(); + } + return elemType; + } + + public Optional getSize() { + if (size == null) { + return Optional.empty(); + } else { + return Optional.of(size); + } + } + + public static MaterializeCompoundDataType create(MaterializeDataType type, int size) { + return new MaterializeCompoundDataType(type, null, size); + } + + public static MaterializeCompoundDataType create(MaterializeDataType type) { + return new MaterializeCompoundDataType(type, null, null); + } +} diff --git a/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java b/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java new file mode 100644 index 000000000..caa801268 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java @@ -0,0 +1,162 @@ +package sqlancer.materialize; + +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; + +public final class MaterializeExpectedValueVisitor implements MaterializeVisitor { + + private final StringBuilder sb = new StringBuilder(); + private static final int NR_TABS = 0; + + private void print(MaterializeExpression expr) { + MaterializeToStringVisitor v = new MaterializeToStringVisitor(); + v.visit(expr); + for (int i = 0; i < NR_TABS; i++) { + sb.append("\t"); + } + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(MaterializeConstant constant) { + print(constant); + } + + @Override + public void visit(MaterializePostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + public String get() { + return sb.toString(); + } + + @Override + public void visit(MaterializeColumnValue c) { + print(c); + } + + @Override + public void visit(MaterializePrefixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(MaterializeSelect op) { + visit(op.getWhereClause()); + } + + @Override + public void visit(MaterializeOrderByTerm op) { + + } + + @Override + public void visit(MaterializeFunction f) { + print(f); + for (int i = 0; i < f.getArguments().length; i++) { + visit(f.getArguments()[i]); + } + } + + @Override + public void visit(MaterializeCastOperation cast) { + print(cast); + visit(cast.getExpression()); + } + + @Override + public void visit(MaterializeBetweenOperation op) { + print(op); + visit(op.getExpr()); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(MaterializeInOperation op) { + print(op); + visit(op.getExpr()); + for (MaterializeExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(MaterializePostfixText op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(MaterializeAggregate op) { + print(op); + for (MaterializeExpression expr : op.getArgs()) { + visit(expr); + } + } + + @Override + public void visit(MaterializeSimilarTo op) { + print(op); + visit(op.getString()); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + } + + @Override + public void visit(MaterializePOSIXRegularExpression op) { + print(op); + visit(op.getString()); + visit(op.getRegex()); + } + + @Override + public void visit(MaterializeFromTable from) { + print(from); + } + + @Override + public void visit(MaterializeSubquery subquery) { + print(subquery); + } + + @Override + public void visit(MaterializeBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(MaterializeLikeOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + +} diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java new file mode 100644 index 000000000..7ab963c03 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -0,0 +1,319 @@ +package sqlancer.materialize; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; + +public class MaterializeGlobalState extends SQLGlobalState { + + public static final char IMMUTABLE = 'i'; + public static final char STABLE = 's'; + public static final char VOLATILE = 'v'; + + private List operators = Collections.emptyList(); + private List collates = Collections.emptyList(); + private List opClasses = Collections.emptyList(); + private List tableAccessMethods = Collections.emptyList(); + // store and allow filtering by function volatility classifications + private final Map functionsAndTypes = new HashMap<>(); + private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + private int lastKnownTableCount; + private int readSchemaCallCount; + + @Override + public void setConnection(SQLConnection con) { + super.setConnection(con); + try { + this.opClasses = getOpclasses(); + this.operators = getOperators(getConnection()); + this.collates = getCollnames(getConnection()); + this.tableAccessMethods = getTableAccessMethods(getConnection()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + private List getCollnames(SQLConnection con) throws SQLException { + List collNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { + while (rs.next()) { + collNames.add(rs.getString(1)); + } + } + } + return collNames; + } + + private List getOpclasses() throws SQLException { + List opClasses = new ArrayList<>(); + // select opcname FROM pg_opclass; + // ERROR: unknown catalog item 'pg_opclass' + opClasses.add("array_ops"); + opClasses.add("array_ops"); + opClasses.add("bit_ops"); + opClasses.add("bool_ops"); + opClasses.add("bpchar_ops"); + opClasses.add("bpchar_ops"); + opClasses.add("bytea_ops"); + opClasses.add("char_ops"); + opClasses.add("char_ops"); + opClasses.add("cidr_ops"); + opClasses.add("cidr_ops"); + opClasses.add("date_ops"); + opClasses.add("date_ops"); + opClasses.add("float4_ops"); + opClasses.add("float4_ops"); + opClasses.add("float8_ops"); + opClasses.add("float8_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("int2_ops"); + opClasses.add("int2_ops"); + opClasses.add("int4_ops"); + opClasses.add("int4_ops"); + opClasses.add("int8_ops"); + opClasses.add("int8_ops"); + opClasses.add("interval_ops"); + opClasses.add("interval_ops"); + opClasses.add("macaddr_ops"); + opClasses.add("macaddr_ops"); + opClasses.add("macaddr8_ops"); + opClasses.add("macaddr8_ops"); + opClasses.add("name_ops"); + opClasses.add("name_ops"); + opClasses.add("numeric_ops"); + opClasses.add("numeric_ops"); + opClasses.add("oid_ops"); + opClasses.add("oid_ops"); + opClasses.add("oidvector_ops"); + opClasses.add("oidvector_ops"); + opClasses.add("record_ops"); + opClasses.add("record_image_ops"); + opClasses.add("text_ops"); + opClasses.add("text_ops"); + opClasses.add("time_ops"); + opClasses.add("time_ops"); + opClasses.add("timestamptz_ops"); + opClasses.add("timestamptz_ops"); + opClasses.add("timetz_ops"); + opClasses.add("timetz_ops"); + opClasses.add("varbit_ops"); + opClasses.add("varchar_ops"); + opClasses.add("varchar_ops"); + opClasses.add("timestamp_ops"); + opClasses.add("timestamp_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("money_ops"); + opClasses.add("bool_ops"); + opClasses.add("bytea_ops"); + opClasses.add("tid_ops"); + opClasses.add("xid_ops"); + opClasses.add("cid_ops"); + opClasses.add("tid_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("aclitem_ops"); + opClasses.add("box_ops"); + opClasses.add("point_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("money_ops"); + opClasses.add("bool_ops"); + opClasses.add("bytea_ops"); + opClasses.add("tid_ops"); + opClasses.add("xid_ops"); + opClasses.add("cid_ops"); + opClasses.add("tid_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("aclitem_ops"); + opClasses.add("box_ops"); + opClasses.add("point_ops"); + opClasses.add("poly_ops"); + opClasses.add("circle_ops"); + opClasses.add("array_ops"); + opClasses.add("uuid_ops"); + opClasses.add("uuid_ops"); + opClasses.add("pg_lsn_ops"); + opClasses.add("pg_lsn_ops"); + opClasses.add("enum_ops"); + opClasses.add("enum_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsquery_ops"); + opClasses.add("tsquery_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("box_ops"); + opClasses.add("quad_point_ops"); + opClasses.add("kd_point_ops"); + opClasses.add("text_ops"); + opClasses.add("poly_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_path_ops"); + opClasses.add("bytea_minmax_ops"); + opClasses.add("char_minmax_ops"); + opClasses.add("name_minmax_ops"); + opClasses.add("int8_minmax_ops"); + opClasses.add("int2_minmax_ops"); + opClasses.add("int4_minmax_ops"); + opClasses.add("text_minmax_ops"); + opClasses.add("oid_minmax_ops"); + opClasses.add("tid_minmax_ops"); + opClasses.add("float4_minmax_ops"); + opClasses.add("float8_minmax_ops"); + opClasses.add("macaddr_minmax_ops"); + opClasses.add("macaddr8_minmax_ops"); + opClasses.add("inet_minmax_ops"); + opClasses.add("inet_inclusion_ops"); + opClasses.add("bpchar_minmax_ops"); + opClasses.add("time_minmax_ops"); + opClasses.add("date_minmax_ops"); + opClasses.add("timestamp_minmax_ops"); + opClasses.add("timestamptz_minmax_ops"); + opClasses.add("interval_minmax_ops"); + opClasses.add("timetz_minmax_ops"); + opClasses.add("bit_minmax_ops"); + opClasses.add("varbit_minmax_ops"); + opClasses.add("numeric_minmax_ops"); + opClasses.add("uuid_minmax_ops"); + opClasses.add("range_inclusion_ops"); + opClasses.add("pg_lsn_minmax_ops"); + opClasses.add("box_inclusion_ops"); + return opClasses; + } + + private List getOperators(SQLConnection con) throws SQLException { + List operators = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { + while (rs.next()) { + operators.add(rs.getString(1)); + } + } + } + return operators; + } + + private List getTableAccessMethods(SQLConnection con) throws SQLException { + List tableAccessMethods = new ArrayList<>(); + try (Statement s = con.createStatement()) { + /* + * pg_am includes both index and table access methods so we need to filter with amtype = 't' + */ + try (ResultSet rs = s.executeQuery("SELECT amname FROM pg_am WHERE amtype = 't';")) { + while (rs.next()) { + tableAccessMethods.add(rs.getString(1)); + } + } + } + return tableAccessMethods; + } + + public List getOperators() { + return operators; + } + + public String getRandomOperator() { + return Randomly.fromList(operators); + } + + public List getCollates() { + return collates; + } + + public String getRandomCollate() { + return Randomly.fromList(collates); + } + + public List getOpClasses() { + return opClasses; + } + + public String getRandomOpclass() { + return Randomly.fromList(opClasses); + } + + public List getTableAccessMethods() { + return tableAccessMethods; + } + + public String getRandomTableAccessMethod() { + return Randomly.fromList(tableAccessMethods); + } + + @Override + public MaterializeSchema readSchema() throws SQLException { + if (MaterializeBugs.bugSchemaReadIncomplete) { + // Workaround for a suspected Materialize bug where tables or columns may be + // missing when reading the schema; retry until stable. + readSchemaCallCount++; + for (int tries = 0; tries < 30; tries++) { + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() + .anyMatch(t -> t.getColumns().isEmpty()); + boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; + boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); + if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; + } + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + } + + public void addFunctionAndType(String functionName, Character functionType) { + this.functionsAndTypes.put(functionName, functionType); + } + + public Map getFunctionsAndTypes() { + return this.functionsAndTypes; + } + + public void setAllowedFunctionTypes(List types) { + this.allowedFunctionTypes = types; + } + + public void setDefaultAllowedFunctionTypes() { + this.allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + } + + public List getAllowedFunctionTypes() { + return this.allowedFunctionTypes; + } + +} diff --git a/src/sqlancer/materialize/MaterializeOptions.java b/src/sqlancer/materialize/MaterializeOptions.java new file mode 100644 index 000000000..104dc9b51 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeOptions.java @@ -0,0 +1,43 @@ +package sqlancer.materialize; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(separators = "=", commandDescription = "Materialize (default port: " + MaterializeOptions.DEFAULT_PORT + + ", default host: " + MaterializeOptions.DEFAULT_HOST + ", default user: " + MaterializeOptions.DEFAULT_USER + + ")") +public class MaterializeOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final String DEFAULT_USER = "materialize"; + public static final int DEFAULT_PORT = 6875; + + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) + public boolean allowBulkInsert; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for Materialize") + public List oracle = Arrays.asList(MaterializeOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) + public boolean testCollations = true; + + @Parameter(names = "--set-max-tables-mvs", description = "Specifies whether to set the maximum number of tables and materialized views intiially", arity = 1) + public boolean setMaxTablesMVs; + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the Materialize server", arity = 1) + public String connectionURL = String.format("postgresql://%s@%s:%d/test", MaterializeOptions.DEFAULT_USER, + MaterializeOptions.DEFAULT_HOST, MaterializeOptions.DEFAULT_PORT); + + @Parameter(names = "--extensions", description = "Specifies a comma-separated list of extension names to be created in each test database", arity = 1) + public String extensions = ""; + + @Override + public List getTestOracleFactory() { + return oracle; + } + +} diff --git a/src/sqlancer/materialize/MaterializeOracleFactory.java b/src/sqlancer/materialize/MaterializeOracleFactory.java new file mode 100644 index 000000000..c652fd56a --- /dev/null +++ b/src/sqlancer/materialize/MaterializeOracleFactory.java @@ -0,0 +1,70 @@ +package sqlancer.materialize; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; +import sqlancer.materialize.oracle.MaterializePivotedQuerySynthesisOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPAggregateOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPHavingOracle; + +public enum MaterializeOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) + .with(MaterializeCommon.getCommonFetchErrors()).with("canceling statement due to statement timeout") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + WHERE { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(MaterializeCommon.getCommonExpressionErrors()).with(MaterializeCommon.getCommonFetchErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + HAVING { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializeTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(new MaterializeTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java new file mode 100644 index 000000000..e7bdb4c4f --- /dev/null +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -0,0 +1,345 @@ +package sqlancer.materialize; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.materialize.gen.MaterializeDeleteGenerator; +import sqlancer.materialize.gen.MaterializeDropIndexGenerator; +import sqlancer.materialize.gen.MaterializeIndexGenerator; +import sqlancer.materialize.gen.MaterializeInsertGenerator; +import sqlancer.materialize.gen.MaterializeTableGenerator; +import sqlancer.materialize.gen.MaterializeUpdateGenerator; +import sqlancer.materialize.gen.MaterializeViewGenerator; + +// EXISTS +// IN +@AutoService(DatabaseProvider.class) +public class MaterializeProvider extends SQLProviderAdapter { + + /** + * Generate only data types and expressions that are understood by PQS. + */ + public static boolean generateOnlyKnown; + + protected String entryURL; + protected String username; + protected String password; + protected String entryPath; + protected String host; + protected int port; + protected String testURL; + protected String databaseName; + protected String createDatabaseCommand; + protected String extensionsList; + + public MaterializeProvider() { + super(MaterializeGlobalState.class, MaterializeOptions.class); + } + + protected MaterializeProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + public enum Action implements AbstractAction { + DELETE(MaterializeDeleteGenerator::create), // + DROP_INDEX(MaterializeDropIndexGenerator::create), // + INSERT(MaterializeInsertGenerator::insert), // + UPDATE(MaterializeUpdateGenerator::create), // + CREATE_INDEX(MaterializeIndexGenerator::generate), // + CREATE_VIEW(MaterializeViewGenerator::create); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(MaterializeGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + protected static int mapActions(MaterializeGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + switch (a) { + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 3); + break; + case DROP_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case DELETE: + nrPerformed = r.getInteger(0, 5); + break; + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + + } + + @Override + public void generateDatabase(MaterializeGlobalState globalState) throws Exception { + readFunctions(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + + extensionsList = globalState.getDbmsSpecificOptions().extensions; + if (!extensionsList.isEmpty()) { + String[] extensionNames = extensionsList.split(","); + + /* + * To avoid of a test interference with an extension objects, create them in a separate schema. Of course, + * they must be truly relocatable. + */ + globalState.executeStatement(new SQLQueryAdapter("CREATE SCHEMA extensions;", true)); + for (int i = 0; i < extensionNames.length; i++) { + globalState.executeStatement(new SQLQueryAdapter( + "CREATE EXTENSION " + extensionNames[i] + " WITH SCHEMA extensions;", true)); + } + } + } + + @Override + public SQLConnection createDatabase(MaterializeGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == MaterializeOracleFactory.PQS)) { + generateOnlyKnown = true; + } + + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + entryPath = "/test"; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; + // trim URL to exclude "jdbc:" + if (entryURL.startsWith("jdbc:")) { + entryURL = entryURL.substring(5); + } + String entryDatabaseName = entryPath.substring(1); + databaseName = globalState.getDatabaseName(); + + try { + URI uri = new URI(entryURL); + String userInfoURI = uri.getUserInfo(); + String pathURI = uri.getPath(); + if (userInfoURI != null) { + // username and password specified in URL take precedence + if (userInfoURI.contains(":")) { + String[] userInfo = userInfoURI.split(":", 2); + username = userInfo[0]; + password = userInfo[1]; + } else { + username = userInfoURI; + password = null; + } + int userInfoIndex = entryURL.indexOf(userInfoURI); + String preUserInfo = entryURL.substring(0, userInfoIndex); + String postUserInfo = entryURL.substring(userInfoIndex + userInfoURI.length() + 1); + entryURL = preUserInfo + postUserInfo; + } + if (pathURI != null) { + entryPath = pathURI; + } + if (host == null) { + host = uri.getHost(); + } + if (port == MainOptions.NO_SET_PORT) { + port = uri.getPort(); + } + entryURL = String.format("%s://%s:%d/%s", uri.getScheme(), host, port, entryDatabaseName); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + if (globalState.getDbmsSpecificOptions().setMaxTablesMVs) { + Connection conMzSystem = DriverManager.getConnection("jdbc:postgresql://localhost:6877/materialize", + "mz_system", "materialize"); + try (Statement s = conMzSystem.createStatement()) { + s.execute("ALTER SYSTEM SET max_tables TO 1000"); + } + try (Statement s = conMzSystem.createStatement()) { + s.execute("ALTER SYSTEM SET max_materialized_views TO 1000"); + } + conMzSystem.close(); + } + int databaseIndex = entryURL.indexOf(entryDatabaseName); + String preDatabaseName = entryURL.substring(0, databaseIndex); + String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); + testURL = preDatabaseName + databaseName + postDatabaseName; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + + con = DriverManager.getConnection("jdbc:" + testURL, username, password); + try (Statement s = con.createStatement()) { + // Serializable transaction isolation is much faster than Strict + // Serializable and should guarantee enough for SQLancer: + // https://materialize.com/docs/overview/isolation-level/ + s.execute("SET transaction_isolation = 'SERIALIZABLE'"); + // Make sure tables still are visible immediately by not using an + // index for them, see + // https://github.com/MaterializeInc/materialize/issues/19431 + s.execute("SET auto_route_introspection_queries = false"); + } + return new SQLConnection(con); + } + + protected void readFunctions(MaterializeGlobalState globalState) throws SQLException { + // ERROR: column "provolatile" does not exist + SQLQueryAdapter query = new SQLQueryAdapter("SELECT proname, 1 FROM pg_proc;"); + SQLancerResultSet rs = query.executeAndGet(globalState); + while (rs.next()) { + String functionName = rs.getString(1); + Character functionType = rs.getString(2).charAt(0); + globalState.addFunctionAndType(functionName, functionType); + } + } + + protected void createTables(MaterializeGlobalState globalState, int numTables) throws Exception { + int existingTables = globalState.getSchema().getDatabaseTables().size(); + int createdTables = 0; + int nextTableIndex = existingTables; + while (existingTables + createdTables < numTables) { + try { + String tableName = DBMSCommon.createTableName(nextTableIndex++); + SQLQueryAdapter createTable = MaterializeTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + if (globalState.executeStatement(createTable)) { + createdTables++; + } + } catch (IgnoreMeException e) { + + } + } + } + + protected void prepareTables(MaterializeGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + MaterializeProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 5000;\n")); + } + + private String getCreateDatabaseCommand(MaterializeGlobalState state) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE DATABASE " + databaseName + " "); + if (Randomly.getBoolean() && ((MaterializeOptions) state.getDbmsSpecificOptions()).testCollations) { + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } + } + } + return sb.toString(); + } + + @Override + public String getDBMSName() { + return "materialize"; + } + + @Override + public String getQueryPlan(String selectStr, MaterializeGlobalState globalState) throws Exception { + String queryPlan = ""; + String explainQuery = "EXPLAIN OPTIMIZED PLAN FOR " + selectStr; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery); + boolean afterProjection = false; // Remove the concrete expression after each Projection operator + SQLancerResultSet rs = q.executeAndGet(globalState); + if (rs != null) { + while (rs.next()) { + String line; + BufferedReader bufReader = new BufferedReader(new StringReader(rs.getString(1))); + while ((line = bufReader.readLine()) != null) { + String targetQueryPlan = line.trim() + ";"; // Unify format + if (targetQueryPlan.startsWith("Explained Query:")) { + continue; + } + if (afterProjection) { + afterProjection = false; + continue; + } + if (targetQueryPlan.startsWith("Project")) { + afterProjection = true; + } + // Remove all concrete expressions by keywords + if (targetQueryPlan.contains(">") || targetQueryPlan.contains("<") || targetQueryPlan.contains("=") + || targetQueryPlan.contains("*") || targetQueryPlan.contains("+") + || targetQueryPlan.contains("'")) { + continue; + } + queryPlan += targetQueryPlan; + } + } + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, MaterializeGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } +} diff --git a/src/sqlancer/materialize/MaterializeSchema.java b/src/sqlancer/materialize/MaterializeSchema.java new file mode 100644 index 000000000..1762c5cb9 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeSchema.java @@ -0,0 +1,330 @@ +package sqlancer.materialize; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.postgresql.util.PSQLException; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTable.TableType; +import sqlancer.materialize.ast.MaterializeConstant; + +public class MaterializeSchema extends AbstractSchema { + + private final String databaseName; + private final List indexNames; + + public List getIndexNames() { + return indexNames; + } + + public enum MaterializeDataType { + INT, BOOLEAN, TEXT, DECIMAL, FLOAT, REAL, BIT; + + public static MaterializeDataType getRandomType() { + List dataTypes = new ArrayList<>(Arrays.asList(values())); + if (MaterializeProvider.generateOnlyKnown) { + dataTypes.remove(MaterializeDataType.DECIMAL); + dataTypes.remove(MaterializeDataType.FLOAT); + dataTypes.remove(MaterializeDataType.REAL); + dataTypes.remove(MaterializeDataType.BIT); + } + return Randomly.fromList(dataTypes); + } + } + + public static class MaterializeColumn extends AbstractTableColumn { + + public MaterializeColumn(String name, MaterializeDataType columnType) { + super(name, null, columnType); + } + + public static MaterializeColumn createDummy(String name) { + return new MaterializeColumn(name, MaterializeDataType.INT); + } + + } + + public static class MaterializeTables extends AbstractTables { + + public MaterializeTables(List tables) { + super(tables); + } + + public MaterializeRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new AssertionError("could not find random row! " + randomRow + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + MaterializeColumn column = getColumns().get(i); + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + MaterializeConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + constant = MaterializeConstant.createNullConstant(); + } else { + switch (column.getType()) { + case INT: + constant = MaterializeConstant.createIntConstant(randomRowValues.getLong(columnIndex)); + break; + case BOOLEAN: + constant = MaterializeConstant + .createBooleanConstant(randomRowValues.getBoolean(columnIndex)); + break; + case TEXT: + constant = MaterializeConstant.createTextConstant(randomRowValues.getString(columnIndex)); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new MaterializeRowValue(this, values); + } catch (PSQLException e) { + throw new IgnoreMeException(); + } + + } + + } + + public static MaterializeDataType getColumnType(String typeString) { + switch (typeString) { + case "smallint": + case "integer": + case "bigint": + return MaterializeDataType.INT; + case "boolean": + return MaterializeDataType.BOOLEAN; + case "text": + case "character": + case "character varying": + case "name": + case "regclass": + return MaterializeDataType.TEXT; + case "numeric": + return MaterializeDataType.DECIMAL; + case "double precision": + return MaterializeDataType.FLOAT; + case "real": + return MaterializeDataType.REAL; + case "bit": + return MaterializeDataType.BIT; + default: + throw new AssertionError(typeString); + } + } + + public static class MaterializeRowValue + extends AbstractRowValue { + + protected MaterializeRowValue(MaterializeTables tables, Map values) { + super(tables, values); + } + + } + + public static class MaterializeTable + extends AbstractRelationalTable { + + public enum TableType { + STANDARD, TEMPORARY + } + + private final TableType tableType; + private final List statistics; + private final boolean isInsertable; + + public MaterializeTable(String tableName, List columns, List indexes, + TableType tableType, List statistics, boolean isView, + boolean isInsertable) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + } + + public List getStatistics() { + return statistics; + } + + public TableType getTableType() { + return tableType; + } + + public boolean isInsertable() { + return isInsertable; + } + + } + + public static final class MaterializeStatisticsObject { + private final String name; + + public MaterializeStatisticsObject(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + public static final class MaterializeIndex extends TableIndex { + + private MaterializeIndex(String indexName) { + super(indexName); + } + + public static MaterializeIndex create(String indexName) { + return new MaterializeIndex(indexName); + } + + @Override + public String getIndexName() { + if (super.getIndexName().contentEquals("PRIMARY")) { + return "`PRIMARY`"; + } else { + return super.getIndexName(); + } + } + + } + + public static MaterializeSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + try { + List databaseTables = new ArrayList<>(); + List indexNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + // ERROR: column "is_insertable_into" does not exist + try (ResultSet rs = s.executeQuery( + "SELECT table_name, table_schema, table_type FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + String tableTypeSchema = rs.getString("table_schema"); + boolean isInsertable = true; + String type = rs.getString("table_type"); + boolean isView = type.equals("VIEW") || type.equals("MATERIALIZED VIEW"); + if (isView) { + isInsertable = false; + } + MaterializeTable.TableType tableType = getTableType(tableTypeSchema); + List databaseColumns = getTableColumns(con, tableName); + List indexes = getIndexes(con, tableName); + List statistics = getStatistics(con); + MaterializeTable t = new MaterializeTable(tableName, databaseColumns, indexes, tableType, + statistics, isView, isInsertable); + for (MaterializeColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT mz_indexes.name, mz_databases.name FROM mz_indexes JOIN mz_relations ON mz_indexes.on_id = mz_relations.id JOIN mz_schemas ON mz_relations.schema_id = mz_schemas.id JOIN mz_databases ON mz_schemas.database_id = mz_databases.id WHERE mz_databases.name = '%s';", + databaseName))) { + while (rs.next()) { + String name = rs.getString(1); + indexNames.add(name); + } + } + } + return new MaterializeSchema(databaseTables, databaseName, indexNames); + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + } + + protected static List getStatistics(SQLConnection con) throws SQLException { + return new ArrayList<>(); + } + + protected static MaterializeTable.TableType getTableType(String tableTypeStr) throws AssertionError { + MaterializeTable.TableType tableType; + if (tableTypeStr.contentEquals("public")) { + tableType = TableType.STANDARD; + } else if (tableTypeStr.startsWith("pg_temp")) { + tableType = TableType.TEMPORARY; + } else { + throw new AssertionError(tableTypeStr); + } + return tableType; + } + + protected static List getIndexes(SQLConnection con, String tableName) throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String + // org.postgresql.util.PSQLException: ERROR: unknown catalog item 'pg_indexes' + .format("SELECT c.relname as indexname FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','I','') AND n.nspname <> 'pg_catalog' AND n.nspname !~ '^pg_toast' AND n.nspname <> 'information_schema' AND c2.relname = '%s' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY indexname;", + tableName))) { + while (rs.next()) { + String indexName = rs.getString("indexname"); + if (DBMSCommon.matchesIndexName(indexName)) { + indexes.add(MaterializeIndex.create(indexName)); + } + } + } + } + return indexes; + } + + protected static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("select column_name, data_type from INFORMATION_SCHEMA.COLUMNS where table_name = '" + + tableName + "' ORDER BY column_name")) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + MaterializeColumn c = new MaterializeColumn(columnName, getColumnType(dataType)); + columns.add(c); + } + } + } + return columns; + } + + public MaterializeSchema(List databaseTables, String databaseName, List indexNames) { + super(databaseTables); + this.databaseName = databaseName; + this.indexNames = indexNames; + } + + public MaterializeTables getRandomTableNonEmptyTables() { + return new MaterializeTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public String getDatabaseName() { + return databaseName; + } + +} diff --git a/src/sqlancer/materialize/MaterializeToStringVisitor.java b/src/sqlancer/materialize/MaterializeToStringVisitor.java new file mode 100644 index 000000000..285be276f --- /dev/null +++ b/src/sqlancer/materialize/MaterializeToStringVisitor.java @@ -0,0 +1,335 @@ +package sqlancer.materialize; + +import java.util.Optional; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; + +public final class MaterializeToStringVisitor extends ToStringVisitor + implements MaterializeVisitor { + + @Override + public void visitSpecific(MaterializeExpression expr) { + MaterializeVisitor.super.visit(expr); + } + + @Override + public void visit(MaterializeConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(MaterializePostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" "); + sb.append(op.getOperatorTextRepresentation()); + } + + @Override + public void visit(MaterializeColumnValue c) { + sb.append(c.getColumn().getFullQualifiedName()); + } + + @Override + public void visit(MaterializePrefixOperation op) { + sb.append(op.getTextRepresentation()); + sb.append(" ("); + visit(op.getExpression()); + sb.append(")"); + } + + @Override + public void visit(MaterializeFromTable from) { + sb.append(from.getTable().getName()); + } + + @Override + public void visit(MaterializeSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(MaterializeSelect s) { + sb.append("SELECT "); + switch (s.getSelectOption()) { + case DISTINCT: + sb.append("DISTINCT "); + if (s.getDistinctOnClause() != null) { + sb.append("ON ("); + visit(s.getDistinctOnClause()); + sb.append(") "); + } + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + visit(s.getFetchColumns()); + sb.append(" FROM "); + visit(s.getFromList()); + + for (MaterializeJoin j : s.getJoinClauses()) { + sb.append(" "); + switch (j.getType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } + sb.append("JOIN"); + break; + case LEFT: + sb.append("LEFT OUTER JOIN"); + break; + case RIGHT: + sb.append("RIGHT OUTER JOIN"); + break; + case FULL: + sb.append("FULL OUTER JOIN"); + break; + case CROSS: + sb.append("CROSS JOIN"); + break; + default: + throw new AssertionError(j.getType()); + } + sb.append(" "); + visit(j.getTableReference()); + if (j.getType() != MaterializeJoinType.CROSS) { + sb.append(" ON "); + visit(j.getOnClause()); + } + } + + if (s.getWhereClause() != null) { + sb.append(" WHERE "); + visit(s.getWhereClause()); + } + if (!s.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(s.getGroupByExpressions()); + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + + } + if (!s.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(s.getOrderByClauses()); + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(MaterializeOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder()); + } + + @Override + public void visit(MaterializeFunction f) { + sb.append(f.getFunctionName()); + sb.append("("); + int i = 0; + for (MaterializeExpression arg : f.getArguments()) { + if (i++ != 0) { + sb.append(", "); + } + visit(arg); + } + sb.append(")"); + } + + @Override + public void visit(MaterializeCastOperation cast) { + if (cast.getCompoundType().getDataType() == MaterializeDataType.BOOLEAN) { + sb.append("("); + MaterializeExpression expr = cast.getExpression(); + visit(expr); + if (expr.getExpressionType() == MaterializeDataType.TEXT) { + sb.append(" != '')"); + } else if (expr.getExpressionType() == MaterializeDataType.BOOLEAN) { + sb.append(" != FALSE)"); + } else { + sb.append(" != 0)"); + } + } else if (Randomly.getBoolean()) { + if (cast.getCompoundType().getDataType() == MaterializeDataType.REAL + || cast.getCompoundType().getDataType() == MaterializeDataType.FLOAT) { + sb.append("CAST(CAST("); + visit(cast.getExpression()); + sb.append(" AS INT) AS "); + appendType(cast); + sb.append(")"); + } else { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + appendType(cast); + sb.append(")"); + } + } else { + if (cast.getCompoundType().getDataType() == MaterializeDataType.REAL + || cast.getCompoundType().getDataType() == MaterializeDataType.FLOAT) { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::INT::"); + appendType(cast); + } else { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::"); + appendType(cast); + } + } + } + + private void appendType(MaterializeCastOperation cast) { + MaterializeCompoundDataType compoundType = cast.getCompoundType(); + switch (compoundType.getDataType()) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: // TODO support also other int types + sb.append("INT"); + break; + case TEXT: + // TODO: append TEXT, CHAR + sb.append(Randomly.fromOptions("VARCHAR")); + break; + case REAL: + sb.append("FLOAT"); + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("REAL"); + break; + case BIT: + sb.append("INT"); + break; + default: + throw new AssertionError(cast.getType()); + } + Optional size = compoundType.getSize(); + if (size.isPresent()) { + sb.append("("); + sb.append(size.get()); + sb.append(")"); + } + } + + @Override + public void visit(MaterializeBetweenOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(") BETWEEN "); + sb.append("("); + visit(op.getLeft()); + sb.append(") AND ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(MaterializeInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN ("); + visit(op.getListElements()); + sb.append(")"); + } + + @Override + public void visit(MaterializePostfixText op) { + visit(op.getExpr()); + sb.append(op.getText()); + } + + @Override + public void visit(MaterializeAggregate op) { + sb.append(op.getFunction()); + sb.append("("); + visit(op.getArgs()); + sb.append(")"); + } + + @Override + public void visit(MaterializeSimilarTo op) { + sb.append("("); + visit(op.getString()); + sb.append(" SIMILAR TO "); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + sb.append(")"); + } + + @Override + public void visit(MaterializePOSIXRegularExpression op) { + visit(op.getString()); + sb.append(op.getOp().getStringRepresentation()); + visit(op.getRegex()); + } + + @Override + public void visit(MaterializeBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + @Override + public void visit(MaterializeLikeOperation op) { + super.visit((BinaryOperation) op); + } + +} diff --git a/src/sqlancer/materialize/MaterializeVisitor.java b/src/sqlancer/materialize/MaterializeVisitor.java new file mode 100644 index 000000000..40ed4e3f2 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeVisitor.java @@ -0,0 +1,127 @@ +package sqlancer.materialize; + +import java.util.List; + +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public interface MaterializeVisitor { + + void visit(MaterializeConstant constant); + + void visit(MaterializePostfixOperation op); + + void visit(MaterializeColumnValue c); + + void visit(MaterializePrefixOperation op); + + void visit(MaterializeSelect op); + + void visit(MaterializeOrderByTerm op); + + void visit(MaterializeFunction f); + + void visit(MaterializeCastOperation cast); + + void visit(MaterializeBetweenOperation op); + + void visit(MaterializeInOperation op); + + void visit(MaterializePostfixText op); + + void visit(MaterializeAggregate op); + + void visit(MaterializeSimilarTo op); + + void visit(MaterializePOSIXRegularExpression op); + + void visit(MaterializeFromTable from); + + void visit(MaterializeSubquery subquery); + + void visit(MaterializeBinaryLogicalOperation op); + + void visit(MaterializeLikeOperation op); + + default void visit(MaterializeExpression expression) { + if (expression instanceof MaterializeConstant) { + visit((MaterializeConstant) expression); + } else if (expression instanceof MaterializePostfixOperation) { + visit((MaterializePostfixOperation) expression); + } else if (expression instanceof MaterializeColumnValue) { + visit((MaterializeColumnValue) expression); + } else if (expression instanceof MaterializePrefixOperation) { + visit((MaterializePrefixOperation) expression); + } else if (expression instanceof MaterializeSelect) { + visit((MaterializeSelect) expression); + } else if (expression instanceof MaterializeOrderByTerm) { + visit((MaterializeOrderByTerm) expression); + } else if (expression instanceof MaterializeFunction) { + visit((MaterializeFunction) expression); + } else if (expression instanceof MaterializeCastOperation) { + visit((MaterializeCastOperation) expression); + } else if (expression instanceof MaterializeBetweenOperation) { + visit((MaterializeBetweenOperation) expression); + } else if (expression instanceof MaterializeInOperation) { + visit((MaterializeInOperation) expression); + } else if (expression instanceof MaterializeAggregate) { + visit((MaterializeAggregate) expression); + } else if (expression instanceof MaterializePostfixText) { + visit((MaterializePostfixText) expression); + } else if (expression instanceof MaterializeSimilarTo) { + visit((MaterializeSimilarTo) expression); + } else if (expression instanceof MaterializePOSIXRegularExpression) { + visit((MaterializePOSIXRegularExpression) expression); + } else if (expression instanceof MaterializeFromTable) { + visit((MaterializeFromTable) expression); + } else if (expression instanceof MaterializeSubquery) { + visit((MaterializeSubquery) expression); + } else if (expression instanceof MaterializeLikeOperation) { + visit((MaterializeLikeOperation) expression); + } else { + throw new AssertionError(expression); + } + } + + static String asString(MaterializeExpression expr) { + MaterializeToStringVisitor visitor = new MaterializeToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(MaterializeExpression expr) { + MaterializeExpectedValueVisitor v = new MaterializeExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + + static String getExpressionAsString(MaterializeGlobalState globalState, MaterializeDataType type, + List columns) { + MaterializeExpression expression = MaterializeExpressionGenerator.generateExpression(globalState, columns, + type); + MaterializeToStringVisitor visitor = new MaterializeToStringVisitor(); + visitor.visit(expression); + return visitor.get(); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeAggregate.java b/src/sqlancer/materialize/ast/MaterializeAggregate.java new file mode 100644 index 000000000..e9dc638f3 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeAggregate.java @@ -0,0 +1,58 @@ +package sqlancer.materialize.ast; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; + +/** + * @see Built-in Aggregate Functions + */ +public class MaterializeAggregate extends FunctionNode + implements MaterializeExpression { + + public enum MaterializeAggregateFunction { + AVG(MaterializeDataType.INT, MaterializeDataType.FLOAT, MaterializeDataType.REAL, MaterializeDataType.DECIMAL), + BIT_AND(MaterializeDataType.INT), BIT_OR(MaterializeDataType.INT), BOOL_AND(MaterializeDataType.BOOLEAN), + BOOL_OR(MaterializeDataType.BOOLEAN), COUNT(MaterializeDataType.INT), MAX, MIN, + SUM(MaterializeDataType.INT, MaterializeDataType.FLOAT, MaterializeDataType.REAL, MaterializeDataType.DECIMAL); + + private MaterializeDataType[] supportedReturnTypes; + + MaterializeAggregateFunction(MaterializeDataType... supportedReturnTypes) { + this.supportedReturnTypes = supportedReturnTypes.clone(); + } + + public List getTypes(MaterializeDataType returnType) { + return Arrays.asList(returnType); + } + + public boolean supportsReturnType(MaterializeDataType returnType) { + return Arrays.asList(supportedReturnTypes).stream().anyMatch(t -> t == returnType) + || supportedReturnTypes.length == 0; + } + + public static List getAggregates(MaterializeDataType type) { + return Arrays.asList(values()).stream().filter(p -> p.supportsReturnType(type)) + .collect(Collectors.toList()); + } + + public MaterializeDataType getRandomReturnType() { + if (supportedReturnTypes.length == 0) { + return Randomly.fromOptions(MaterializeDataType.getRandomType()); + } else { + return Randomly.fromOptions(supportedReturnTypes); + } + } + + } + + public MaterializeAggregate(List args, MaterializeAggregateFunction func) { + super(func, args); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeAlias.java b/src/sqlancer/materialize/ast/MaterializeAlias.java new file mode 100644 index 000000000..ec5445125 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeAlias.java @@ -0,0 +1,35 @@ +package sqlancer.materialize.ast; + +import sqlancer.common.visitor.UnaryOperation; + +public class MaterializeAlias implements UnaryOperation, MaterializeExpression { + + private final MaterializeExpression expr; + private final String alias; + + public MaterializeAlias(MaterializeExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + @Override + public MaterializeExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return " as " + alias; + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java b/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java new file mode 100644 index 000000000..dcb7b7cdb --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java @@ -0,0 +1,66 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; + +public final class MaterializeBetweenOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final MaterializeExpression left; + private final MaterializeExpression right; + private final boolean isSymmetric; + + public MaterializeBetweenOperation(MaterializeExpression expr, MaterializeExpression left, + MaterializeExpression right, boolean symmetric) { + this.expr = expr; + this.left = left; + this.right = right; + isSymmetric = symmetric; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public MaterializeExpression getLeft() { + return left; + } + + public MaterializeExpression getRight() { + return right; + } + + public boolean isSymmetric() { + return isSymmetric; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeBinaryComparisonOperation leftComparison = new MaterializeBinaryComparisonOperation(left, expr, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryComparisonOperation rightComparison = new MaterializeBinaryComparisonOperation(expr, right, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryLogicalOperation andOperation = new MaterializeBinaryLogicalOperation(leftComparison, + rightComparison, MaterializeBinaryLogicalOperation.BinaryLogicalOperator.AND); + if (isSymmetric) { + MaterializeBinaryComparisonOperation leftComparison2 = new MaterializeBinaryComparisonOperation(right, expr, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryComparisonOperation rightComparison2 = new MaterializeBinaryComparisonOperation(expr, left, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryLogicalOperation andOperation2 = new MaterializeBinaryLogicalOperation(leftComparison2, + rightComparison2, MaterializeBinaryLogicalOperation.BinaryLogicalOperator.AND); + MaterializeBinaryLogicalOperation orOp = new MaterializeBinaryLogicalOperation(andOperation, andOperation2, + BinaryLogicalOperator.OR); + return orOp.getExpectedValue(); + } else { + return andOperation.getExpectedValue(); + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java new file mode 100644 index 000000000..0ba48924f --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java @@ -0,0 +1,103 @@ +package sqlancer.materialize.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation.MaterializeBinaryOperator; + +public class MaterializeBinaryArithmeticOperation + extends BinaryOperatorNode implements MaterializeExpression { + + public enum MaterializeBinaryOperator implements Operator { + + ADDITION("+") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l + r); + } + + }, + SUBTRACTION("-") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + + } + + }, + MODULO("%") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + + } + }; + + private String textRepresentation; + + private static MaterializeConstant applyBitOperation(MaterializeConstant left, MaterializeConstant right, + BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + long leftVal = left.cast(MaterializeDataType.INT).asInt(); + long rightVal = right.cast(MaterializeDataType.INT).asInt(); + long value = op.apply(leftVal, rightVal); + return MaterializeConstant.createIntConstant(value); + } + } + + MaterializeBinaryOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right); + + public static MaterializeBinaryOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public MaterializeBinaryArithmeticOperation(MaterializeExpression left, MaterializeExpression right, + MaterializeBinaryOperator op) { + super(left, right, op); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpected = getLeft().getExpectedValue(); + MaterializeConstant rightExpected = getRight().getExpectedValue(); + if (leftExpected == null || rightExpected == null) { + return null; + } + return getOp().apply(leftExpected, rightExpected); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java new file mode 100644 index 000000000..8f143da76 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java @@ -0,0 +1,46 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation.MaterializeBinaryBitOperator; + +public class MaterializeBinaryBitOperation extends + BinaryOperatorNode implements MaterializeExpression { + + public enum MaterializeBinaryBitOperator implements Operator { + BITWISE_AND("&"), // + BITWISE_OR("|"), // + BITWISE_XOR("#"), // + BITWISE_SHIFT_LEFT("<<"), // + BITWISE_SHIFT_RIGHT(">>"); + + private String text; + + MaterializeBinaryBitOperator(String text) { + this.text = text; + } + + public static MaterializeBinaryBitOperator getRandom() { + return Randomly.fromOptions(MaterializeBinaryBitOperator.values()); + } + + @Override + public String getTextRepresentation() { + return text; + } + + } + + public MaterializeBinaryBitOperation(MaterializeBinaryBitOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right, op); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java new file mode 100644 index 000000000..25121935a --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java @@ -0,0 +1,121 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator; + +public class MaterializeBinaryComparisonOperation + extends BinaryOperatorNode + implements MaterializeExpression { + + public enum MaterializeBinaryComparisonOperator implements Operator { + EQUALS("=") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + NOT_EQUALS("!=") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.isBoolean()) { + return MaterializeConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + LESS("<") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan.isBoolean() && !lessThan.asBoolean()) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return MaterializeConstant.createFalse(); + } else { + MaterializeConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializePrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return MaterializeConstant.createTrue(); + } else { + MaterializeConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializePrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + + }; + + private final String textRepresentation; + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + MaterializeBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public abstract MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal); + + public static MaterializeBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(MaterializeBinaryComparisonOperator.values()); + } + + } + + public MaterializeBinaryComparisonOperation(MaterializeExpression left, MaterializeExpression right, + MaterializeBinaryComparisonOperator op) { + super(left, right, op); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java new file mode 100644 index 000000000..8b8e457e2 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java @@ -0,0 +1,89 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; + +public class MaterializeBinaryLogicalOperation extends BinaryOperatorNode + implements MaterializeExpression { + + public enum BinaryLogicalOperator implements Operator { + AND { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + MaterializeConstant leftBool = left.cast(MaterializeDataType.BOOLEAN); + MaterializeConstant rightBool = right.cast(MaterializeDataType.BOOLEAN); + if (leftBool.isNull()) { + if (rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + if (rightBool.asBoolean()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createFalse(); + } + } + } else if (!leftBool.asBoolean()) { + return MaterializeConstant.createFalse(); + } + assert leftBool.asBoolean(); + if (rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createBooleanConstant(rightBool.isBoolean() && rightBool.asBoolean()); + } + } + }, + OR { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + MaterializeConstant leftBool = left.cast(MaterializeDataType.BOOLEAN); + MaterializeConstant rightBool = right.cast(MaterializeDataType.BOOLEAN); + if (leftBool.isBoolean() && leftBool.asBoolean()) { + return MaterializeConstant.createTrue(); + } + if (rightBool.isBoolean() && rightBool.asBoolean()) { + return MaterializeConstant.createTrue(); + } + if (leftBool.isNull() || rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializeConstant.createFalse(); + } + }; + + public abstract MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right); + + public static BinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializeBinaryLogicalOperation(MaterializeExpression left, MaterializeExpression right, + BinaryLogicalOperator op) { + super(left, right, op); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java new file mode 100644 index 000000000..cf1ef66ef --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java @@ -0,0 +1,74 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeBinaryRangeOperation extends BinaryNode + implements MaterializeExpression { + + private final String op; + + public enum MaterializeBinaryRangeOperator implements Operator { + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); + + private final String textRepresentation; + + MaterializeBinaryRangeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public static MaterializeBinaryRangeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum MaterializeBinaryRangeComparisonOperator { + CONTAINS_RANGE_OR_ELEMENT("@>"), RANGE_OR_ELEMENT_IS_CONTAINED("<@"), OVERLAP("&&"), STRICT_LEFT_OF("<<"), + STRICT_RIGHT_OF(">>"); + + private final String textRepresentation; + + MaterializeBinaryRangeComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + public static MaterializeBinaryRangeComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public MaterializeBinaryRangeOperation(MaterializeBinaryRangeComparisonOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + public MaterializeBinaryRangeOperation(MaterializeBinaryRangeOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public String getOperatorRepresentation() { + return op; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeCastOperation.java b/src/sqlancer/materialize/ast/MaterializeCastOperation.java new file mode 100644 index 000000000..ff7e20180 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeCastOperation.java @@ -0,0 +1,45 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeCastOperation implements MaterializeExpression { + + private final MaterializeExpression expression; + private final MaterializeCompoundDataType type; + + public MaterializeCastOperation(MaterializeExpression expression, MaterializeCompoundDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + @Override + public MaterializeDataType getExpressionType() { + return type.getDataType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expression.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type.getDataType()); + } + + public MaterializeExpression getExpression() { + return expression; + } + + public MaterializeDataType getType() { + return type.getDataType(); + } + + public MaterializeCompoundDataType getCompoundType() { + return type; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeColumnValue.java b/src/sqlancer/materialize/ast/MaterializeColumnValue.java new file mode 100644 index 000000000..054f34ffd --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeColumnValue.java @@ -0,0 +1,34 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeColumnValue implements MaterializeExpression { + + private final MaterializeColumn c; + private final MaterializeConstant expectedValue; + + public MaterializeColumnValue(MaterializeColumn c, MaterializeConstant expectedValue) { + this.c = c; + this.expectedValue = expectedValue; + } + + @Override + public MaterializeDataType getExpressionType() { + return c.getType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + return expectedValue; + } + + public static MaterializeColumnValue create(MaterializeColumn c, MaterializeConstant expected) { + return new MaterializeColumnValue(c, expected); + } + + public MaterializeColumn getColumn() { + return c; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeConcatOperation.java b/src/sqlancer/materialize/ast/MaterializeConcatOperation.java new file mode 100644 index 000000000..c1963c871 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeConcatOperation.java @@ -0,0 +1,37 @@ +package sqlancer.materialize.ast; + +import sqlancer.common.ast.BinaryNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeConcatOperation extends BinaryNode implements MaterializeExpression { + + public MaterializeConcatOperation(MaterializeExpression left, MaterializeExpression right) { + super(left, right); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.TEXT; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + if (leftExpectedValue.isNull() || rightExpectedValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } + String leftStr = leftExpectedValue.cast(MaterializeDataType.TEXT).getUnquotedTextRepresentation(); + String rightStr = rightExpectedValue.cast(MaterializeDataType.TEXT).getUnquotedTextRepresentation(); + return MaterializeConstant.createTextConstant(leftStr + rightStr); + } + + @Override + public String getOperatorRepresentation() { + return "||"; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeConstant.java b/src/sqlancer/materialize/ast/MaterializeConstant.java new file mode 100644 index 000000000..dbf668806 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeConstant.java @@ -0,0 +1,516 @@ +package sqlancer.materialize.ast; + +import java.math.BigDecimal; + +import sqlancer.IgnoreMeException; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public abstract class MaterializeConstant implements MaterializeExpression { + + public abstract String getTextRepresentation(); + + public abstract String getUnquotedTextRepresentation(); + + public static class BooleanConstant extends MaterializeConstant { + + private final boolean value; + + public BooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return value ? "TRUE" : "FALSE"; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return MaterializeConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else if (rightVal.isString()) { + return MaterializeConstant + .createBooleanConstant(value == rightVal.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isString()) { + return isLessThan(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else { + assert rightVal.isBoolean(); + return MaterializeConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + return this; + case INT: + return MaterializeConstant.createIntConstant(value ? 1 : 0); + case TEXT: + return MaterializeConstant.createTextConstant(value ? "true" : "false"); + default: + return null; + } + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static class MaterializeNullConstant extends MaterializeConstant { + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + return MaterializeConstant.createNullConstant(); + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + return MaterializeConstant.createNullConstant(); + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + return MaterializeConstant.createNullConstant(); + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static class StringConstant extends MaterializeConstant { + + private final String value; + + public StringConstant(String value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'", value.replace("'", "''")); + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(MaterializeDataType.INT).isEquals(rightVal.cast(MaterializeDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isEquals(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(MaterializeDataType.INT).isLessThan(rightVal.cast(MaterializeDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isLessThan(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + if (type == MaterializeDataType.TEXT) { + return this; + } + String s = value.trim(); + switch (type) { + case BOOLEAN: + try { + return MaterializeConstant.createBooleanConstant(Long.parseLong(s) != 0); + } catch (NumberFormatException e) { + } + switch (s.toUpperCase()) { + case "T": + case "TR": + case "TRU": + case "TRUE": + case "1": + case "YES": + case "YE": + case "Y": + case "ON": + return MaterializeConstant.createTrue(); + case "F": + case "FA": + case "FAL": + case "FALS": + case "FALSE": + case "N": + case "NO": + case "OF": + case "OFF": + default: + return MaterializeConstant.createFalse(); + } + case INT: + try { + return MaterializeConstant.createIntConstant(Long.parseLong(s)); + } catch (NumberFormatException e) { + return MaterializeConstant.createIntConstant(-1); + } + case TEXT: + return this; + default: + return null; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.TEXT; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public String asString() { + return value; + } + + @Override + public String getUnquotedTextRepresentation() { + return value; + } + + } + + public static class IntConstant extends MaterializeConstant { + + private final long val; + + public IntConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + public long asInt() { + return val; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isEquals(rightVal); + } else if (rightVal.isInt()) { + return MaterializeConstant.createBooleanConstant(val == rightVal.asInt()); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(val == rightVal.cast(MaterializeDataType.INT).asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return MaterializeConstant.createBooleanConstant(val < rightVal.asInt()); + } else if (rightVal.isBoolean()) { + throw new AssertionError(rightVal); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(val < rightVal.cast(MaterializeDataType.INT).asInt()); + } else { + throw new IgnoreMeException(); + } + + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + return MaterializeConstant.createBooleanConstant(val != 0); + case INT: + return this; + case TEXT: + return MaterializeConstant.createTextConstant(String.valueOf(val)); + default: + return null; + } + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static MaterializeConstant createNullConstant() { + return new MaterializeNullConstant(); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isString() { + return false; + } + + public static MaterializeConstant createIntConstant(long val) { + return new IntConstant(val); + } + + public static MaterializeConstant createBooleanConstant(boolean val) { + return new BooleanConstant(val); + } + + @Override + public MaterializeConstant getExpectedValue() { + return this; + } + + public boolean isNull() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public static MaterializeConstant createFalse() { + return createBooleanConstant(false); + } + + public static MaterializeConstant createTrue() { + return createBooleanConstant(true); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isBoolean() { + return false; + } + + public abstract MaterializeConstant isEquals(MaterializeConstant rightVal); + + public boolean isInt() { + return false; + } + + protected abstract MaterializeConstant isLessThan(MaterializeConstant rightVal); + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract MaterializeConstant cast(MaterializeDataType type); + + public static MaterializeConstant createTextConstant(String string) { + return new StringConstant(string); + } + + public abstract static class MaterializeConstantBase extends MaterializeConstant { + + @Override + public String getUnquotedTextRepresentation() { + return null; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + return null; + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + return null; + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + return null; + } + } + + public static class DecimalConstant extends MaterializeConstantBase { + + private final BigDecimal val; + + public DecimalConstant(BigDecimal val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.DECIMAL; + } + + } + + public static class FloatConstant extends MaterializeConstantBase { + + private final float val; + + public FloatConstant(float val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.FLOAT; + } + + } + + public static class DoubleConstant extends MaterializeConstantBase { + + private final double val; + + public DoubleConstant(double val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.FLOAT; + } + + } + + public static class BitConstant extends MaterializeConstantBase { + + private final long val; + + public BitConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.format("%d", val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BIT; + } + + } + + public static MaterializeConstant createDecimalConstant(BigDecimal bigDecimal) { + return new DecimalConstant(bigDecimal); + } + + public static MaterializeConstant createFloatConstant(float val) { + return new FloatConstant(val); + } + + public static MaterializeConstant createDoubleConstant(double val) { + return new DoubleConstant(val); + } + + public static MaterializeExpression createBitConstant(long integer) { + return new BitConstant(integer); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeExpression.java b/src/sqlancer/materialize/ast/MaterializeExpression.java new file mode 100644 index 000000000..70444d8ee --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeExpression.java @@ -0,0 +1,16 @@ +package sqlancer.materialize.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public interface MaterializeExpression extends Expression { + + default MaterializeDataType getExpressionType() { + return null; + } + + default MaterializeConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializeFunction.java b/src/sqlancer/materialize/ast/MaterializeFunction.java new file mode 100644 index 000000000..24fdce866 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeFunction.java @@ -0,0 +1,256 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeFunction implements MaterializeExpression { + + private final String func; + private final MaterializeExpression[] args; + private final MaterializeDataType returnType; + private MaterializeFunctionWithResult functionWithKnownResult; + + public MaterializeFunction(MaterializeFunctionWithResult func, MaterializeDataType returnType, + MaterializeExpression... args) { + functionWithKnownResult = func; + this.func = func.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public MaterializeFunction(MaterializeFunctionWithUnknownResult f, MaterializeDataType returnType, + MaterializeExpression... args) { + this.func = f.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public String getFunctionName() { + return func; + } + + public MaterializeExpression[] getArguments() { + return args.clone(); + } + + public enum MaterializeFunctionWithResult { + ABS(1, "abs") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant + .createIntConstant(Math.abs(evaluatedArgs[0].cast(MaterializeDataType.INT).asInt())); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { returnType }; + } + + }, + LOWER(1, "lower") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createTextConstant(text.toLowerCase()); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.TEXT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + + }, + LENGTH(1, "length") { + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createIntConstant(text.length()); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + }, + UPPER(1, "upper") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createTextConstant(text.toUpperCase()); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.TEXT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + + }, + NUM_NONNULLS(1, "num_nonnulls") { + @Override + public MaterializeConstant apply(MaterializeConstant[] args, MaterializeExpression... origArgs) { + int nr = 0; + for (MaterializeConstant c : args) { + if (!c.isNull()) { + nr++; + } + } + return MaterializeConstant.createIntConstant(nr); + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + NUM_NULLS(1, "num_nulls") { + @Override + public MaterializeConstant apply(MaterializeConstant[] args, MaterializeExpression... origArgs) { + int nr = 0; + for (MaterializeConstant c : args) { + if (c.isNull()) { + nr++; + } + } + return MaterializeConstant.createIntConstant(nr); + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }; + + private String functionName; + final int nrArgs; + private final boolean variadic; + + public MaterializeDataType[] getRandomTypes(int nr) { + MaterializeDataType[] types = new MaterializeDataType[nr]; + for (int i = 0; i < types.length; i++) { + types[i] = MaterializeDataType.getRandomType(); + } + return types; + } + + MaterializeFunctionWithResult(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + /** + * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number + * of arguments is returned. + * + * @return the number of arguments + */ + public int getNrArgs() { + return nrArgs; + } + + public abstract MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args); + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + + public abstract boolean supportsReturnType(MaterializeDataType type); + + public abstract MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, + int nrArguments); + + public boolean checkArguments(MaterializeExpression... constants) { + return true; + } + + } + + @Override + public MaterializeConstant getExpectedValue() { + if (functionWithKnownResult == null) { + return null; + } + MaterializeConstant[] constants = new MaterializeConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + if (constants[i] == null) { + return null; + } + } + return functionWithKnownResult.apply(constants, args); + } + + @Override + public MaterializeDataType getExpressionType() { + return returnType; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java b/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java new file mode 100644 index 000000000..fb34c2247 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java @@ -0,0 +1,148 @@ +package sqlancer.materialize.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public enum MaterializeFunctionWithUnknownResult { + + CURRENT_DATABASE("current_database", MaterializeDataType.TEXT), // name + CURRENT_SCHEMA("current_schema", MaterializeDataType.TEXT), // name + PG_BACKEND_PID("pg_backend_pid", MaterializeDataType.INT), + PG_CURRENT_LOGFILE("pg_current_logfile", MaterializeDataType.TEXT), + PG_IS_OTHER_TEMP_SCHEMA("pg_is_other_temp_schema", MaterializeDataType.BOOLEAN), + PG_JIT_AVAILABLE("pg_jit_available", MaterializeDataType.BOOLEAN), + PG_NOTIFICATION_QUEUE_USAGE("pg_notification_queue_usage", MaterializeDataType.REAL), + PG_TRIGGER_DEPTH("pg_trigger_depth", MaterializeDataType.INT), VERSION("version", MaterializeDataType.TEXT), + + // + TO_CHAR("to_char", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT) { + @Override + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = super.getArguments(returnType, gen, depth); + args[0] = gen.generateExpression(MaterializeDataType.getRandomType()); + return args; + } + }, + + // String functions + ASCII("ascii", MaterializeDataType.INT, MaterializeDataType.TEXT), + BTRIM("btrim", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + CHR("chr", MaterializeDataType.TEXT, MaterializeDataType.INT), + CONVERT_FROM("convert_from", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT) { + @Override + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = super.getArguments(returnType, gen, depth); + args[1] = MaterializeConstant.createTextConstant(Randomly.fromOptions("UTF8", "LATIN1")); + return args; + } + }, + INITCAP("initcap", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + LEFT("left", MaterializeDataType.TEXT, MaterializeDataType.INT, MaterializeDataType.TEXT), + LOWER("lower", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + MD5("md5", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + UPPER("upper", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + QUOTE_LITERAL("quote_literal", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + QUOTE_IDENT("quote_ident", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REGEX_REPLACE("regex_replace", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REPLACE("replace", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REVERSE("reverse", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + RIGHT("right", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT), + RPAD("rpad", MaterializeDataType.TEXT, MaterializeDataType.INT, MaterializeDataType.TEXT), + RTRIM("rtrim", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + SPLIT_PART("split_part", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT), + STRPOS("strpos", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + SUBSTR("substr", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT, + MaterializeDataType.INT), + TO_ASCII("to_ascii", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + TO_HEX("to_hex", MaterializeDataType.INT, MaterializeDataType.TEXT), + TRANSLATE("translate", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT, + MaterializeDataType.TEXT), + // mathematical functions + ABS("abs", MaterializeDataType.REAL, MaterializeDataType.REAL), + CBRT("cbrt", MaterializeDataType.REAL, MaterializeDataType.REAL), CEILING("ceiling", MaterializeDataType.REAL), // + DEGREES("degrees", MaterializeDataType.REAL), EXP("exp", MaterializeDataType.REAL), + LN("ln", MaterializeDataType.REAL), LOG("log", MaterializeDataType.REAL), + LOG2("log", MaterializeDataType.REAL, MaterializeDataType.REAL), PI("pi", MaterializeDataType.REAL), + POWER("power", MaterializeDataType.REAL, MaterializeDataType.REAL), + TRUNC("trunc", MaterializeDataType.REAL, MaterializeDataType.INT), + TRUNC2("trunc", MaterializeDataType.REAL, MaterializeDataType.INT, MaterializeDataType.REAL), + FLOOR("floor", MaterializeDataType.REAL), + + // trigonometric functions - complete + ACOS("acos", MaterializeDataType.REAL), // + ACOSD("acosd", MaterializeDataType.REAL), // + ASIN("asin", MaterializeDataType.REAL), // + ASIND("asind", MaterializeDataType.REAL), // + ATAN("atan", MaterializeDataType.REAL), // + ATAND("atand", MaterializeDataType.REAL), // + ATAN2("atan2", MaterializeDataType.REAL, MaterializeDataType.REAL), // + ATAN2D("atan2d", MaterializeDataType.REAL, MaterializeDataType.REAL), // + COS("cos", MaterializeDataType.REAL), // + COSD("cosd", MaterializeDataType.REAL), // + COT("cot", MaterializeDataType.REAL), // + COTD("cotd", MaterializeDataType.REAL), // + SIN("sin", MaterializeDataType.REAL), // + SIND("sind", MaterializeDataType.REAL), // + TAN("tan", MaterializeDataType.REAL), // + TAND("tand", MaterializeDataType.REAL), // + + // hyperbolic functions - complete + SINH("sinh", MaterializeDataType.REAL), // + COSH("cosh", MaterializeDataType.REAL), // + TANH("tanh", MaterializeDataType.REAL), // + ASINH("asinh", MaterializeDataType.REAL), // + ACOSH("acosh", MaterializeDataType.REAL), // + ATANH("atanh", MaterializeDataType.REAL), // + + GET_BIT("get_bit", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.INT), + GET_BYTE("get_byte", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.INT), + + GET_COLUMN_SIZE("get_column_size", MaterializeDataType.INT, MaterializeDataType.TEXT); + + private String functionName; + private MaterializeDataType returnType; + private MaterializeDataType[] argTypes; + + MaterializeFunctionWithUnknownResult(String functionName, MaterializeDataType returnType, + MaterializeDataType... indexType) { + this.functionName = functionName; + this.returnType = returnType; + this.argTypes = indexType.clone(); + + } + + public boolean isCompatibleWithReturnType(MaterializeDataType t) { + return t == returnType; + } + + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = new MaterializeExpression[argTypes.length]; + for (int i = 0; i < args.length; i++) { + args[i] = gen.generateExpression(depth, argTypes[i]); + } + return args; + + } + + public String getName() { + return functionName; + } + + public static List getSupportedFunctions(MaterializeDataType type) { + List functions = new ArrayList<>(); + for (MaterializeFunctionWithUnknownResult func : values()) { + if (func.isCompatibleWithReturnType(type)) { + functions.add(func); + } + } + return functions; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeInOperation.java b/src/sqlancer/materialize/ast/MaterializeInOperation.java new file mode 100644 index 000000000..270a973dc --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeInOperation.java @@ -0,0 +1,66 @@ +package sqlancer.materialize.ast; + +import java.util.List; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeInOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final List listElements; + private final boolean isTrue; + + public MaterializeInOperation(MaterializeExpression expr, List listElements, + boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftValue = expr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } + boolean isNull = false; + for (MaterializeExpression expr : getListElements()) { + MaterializeConstant rightExpectedValue = expr.getExpectedValue(); + if (rightExpectedValue == null) { + return null; + } + if (rightExpectedValue.isNull()) { + isNull = true; + } else if (rightExpectedValue.isEquals(this.expr.getExpectedValue()).isBoolean() + && rightExpectedValue.isEquals(this.expr.getExpectedValue()).asBoolean()) { + return MaterializeConstant.createBooleanConstant(isTrue); + } + } + + if (isNull) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createBooleanConstant(!isTrue); + } + } + + public boolean isTrue() { + return isTrue; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializeJoin.java b/src/sqlancer/materialize/ast/MaterializeJoin.java new file mode 100644 index 000000000..ef8f1ac61 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeJoin.java @@ -0,0 +1,58 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public class MaterializeJoin + implements MaterializeExpression, Join { + + public enum MaterializeJoinType { + INNER, LEFT, RIGHT, FULL, CROSS; + + public static MaterializeJoinType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + private MaterializeExpression onClause; + private final MaterializeExpression tableReference; + private final MaterializeJoinType type; + + public MaterializeJoin(MaterializeExpression tableReference, MaterializeExpression onClause, + MaterializeJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; + } + + public MaterializeExpression getTableReference() { + return tableReference; + } + + public MaterializeExpression getOnClause() { + return onClause; + } + + public MaterializeJoinType getType() { + return type; + } + + @Override + public MaterializeDataType getExpressionType() { + throw new AssertionError(); + } + + @Override + public MaterializeConstant getExpectedValue() { + throw new AssertionError(); + } + + @Override + public void setOnClause(MaterializeExpression onClause) { + this.onClause = onClause; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializeLikeOperation.java b/src/sqlancer/materialize/ast/MaterializeLikeOperation.java new file mode 100644 index 000000000..b14209993 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeLikeOperation.java @@ -0,0 +1,38 @@ +package sqlancer.materialize.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.common.ast.BinaryNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeLikeOperation extends BinaryNode implements MaterializeExpression { + + public MaterializeLikeOperation(MaterializeExpression left, MaterializeExpression right) { + super(left, right); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftVal = getLeft().getExpectedValue(); + MaterializeConstant rightVal = getRight().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + if (leftVal.isNull() || rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + boolean val = LikeImplementationHelper.match(leftVal.asString(), rightVal.asString(), 0, 0, true); + return MaterializeConstant.createBooleanConstant(val); + } + } + + @Override + public String getOperatorRepresentation() { + return "LIKE"; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java b/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java new file mode 100644 index 000000000..d4704c87f --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java @@ -0,0 +1,42 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeOrderByTerm implements MaterializeExpression { + + private final MaterializeOrder order; + private final MaterializeExpression expr; + + public enum MaterializeOrder { + ASC, DESC; + + public static MaterializeOrder getRandomOrder() { + return Randomly.fromOptions(MaterializeOrder.values()); + } + } + + public MaterializeOrderByTerm(MaterializeExpression expr, MaterializeOrder order) { + this.expr = expr; + this.order = order; + } + + public MaterializeOrder getOrder() { + return order; + } + + public MaterializeExpression getExpr() { + return expr; + } + + @Override + public MaterializeConstant getExpectedValue() { + throw new AssertionError(this); + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java b/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java new file mode 100644 index 000000000..127bafe79 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java @@ -0,0 +1,65 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePOSIXRegularExpression implements MaterializeExpression { + + private MaterializeExpression string; + private MaterializeExpression regex; + private POSIXRegex op; + + public enum POSIXRegex implements Operator { + MATCH_CASE_SENSITIVE("~"), MATCH_CASE_INSENSITIVE("~*"), NOT_MATCH_CASE_SENSITIVE("!~"), + NOT_MATCH_CASE_INSENSITIVE("!~*"); + + private String repr; + + POSIXRegex(String repr) { + this.repr = repr; + } + + public String getStringRepresentation() { + return repr; + } + + public static POSIXRegex getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializePOSIXRegularExpression(MaterializeExpression string, MaterializeExpression regex, POSIXRegex op) { + this.string = string; + this.regex = regex; + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + return null; + } + + public MaterializeExpression getRegex() { + return regex; + } + + public MaterializeExpression getString() { + return string; + } + + public POSIXRegex getOp() { + return op; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePostfixOperation.java b/src/sqlancer/materialize/ast/MaterializePostfixOperation.java new file mode 100644 index 000000000..7bee2e38c --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePostfixOperation.java @@ -0,0 +1,151 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePostfixOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final PostfixOperator op; + private final String operatorTextRepresentation; + + public enum PostfixOperator implements Operator { + IS_NULL("IS NULL", "ISNULL") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return MaterializeDataType.values(); + } + + }, + IS_UNKNOWN("IS UNKNOWN") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + }, + + IS_NOT_NULL("IS NOT NULL") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return MaterializeDataType.values(); + } + + }, + IS_NOT_UNKNOWN("IS NOT UNKNOWN") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + }, + IS_TRUE("IS TRUE") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createFalse(); + } else { + return MaterializeConstant + .createBooleanConstant(expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + + }, + IS_FALSE("IS FALSE") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createFalse(); + } else { + return MaterializeConstant + .createBooleanConstant(!expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + + }; + + private String[] textRepresentations; + + PostfixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public abstract MaterializeConstant apply(MaterializeConstant expectedValue); + + public abstract MaterializeDataType[] getInputDataTypes(); + + public static PostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializePostfixOperation(MaterializeExpression expr, PostfixOperator op) { + this.expr = expr; + this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.apply(expectedValue); + } + + public String getOperatorTextRepresentation() { + return operatorTextRepresentation; + } + + public static MaterializeExpression create(MaterializeExpression expr, PostfixOperator op) { + return new MaterializePostfixOperation(expr, op); + } + + public MaterializeExpression getExpression() { + return expr; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePostfixText.java b/src/sqlancer/materialize/ast/MaterializePostfixText.java new file mode 100644 index 000000000..a64b03c6d --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePostfixText.java @@ -0,0 +1,37 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePostfixText implements MaterializeExpression { + + private final MaterializeExpression expr; + private final String text; + private final MaterializeConstant expectedValue; + private final MaterializeDataType type; + + public MaterializePostfixText(MaterializeExpression expr, String text, MaterializeConstant expectedValue, + MaterializeDataType type) { + this.expr = expr; + this.text = text; + this.expectedValue = expectedValue; + this.type = type; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + @Override + public MaterializeConstant getExpectedValue() { + return expectedValue; + } + + @Override + public MaterializeDataType getExpressionType() { + return type; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializePrefixOperation.java b/src/sqlancer/materialize/ast/MaterializePrefixOperation.java new file mode 100644 index 000000000..456c65b45 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePrefixOperation.java @@ -0,0 +1,119 @@ +package sqlancer.materialize.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePrefixOperation implements MaterializeExpression { + + public enum PrefixOperator implements Operator { + NOT("NOT", MaterializeDataType.BOOLEAN) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant + .createBooleanConstant(!expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + }, + UNARY_PLUS("+", MaterializeDataType.INT) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + // TODO: actual converts to double precision + return expectedValue; + } + + }, + UNARY_MINUS("-", MaterializeDataType.INT) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + // TODO + throw new IgnoreMeException(); + } + if (expectedValue.isInt() && expectedValue.asInt() == Long.MIN_VALUE) { + throw new IgnoreMeException(); + } + try { + return MaterializeConstant.createIntConstant(-expectedValue.asInt()); + } catch (UnsupportedOperationException e) { + return null; + } + } + + }; + + private String textRepresentation; + private MaterializeDataType[] dataTypes; + + PrefixOperator(String textRepresentation, MaterializeDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract MaterializeDataType getExpressionType(); + + protected abstract MaterializeConstant getExpectedValue(MaterializeConstant expectedValue); + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + + private final MaterializeExpression expr; + private final PrefixOperator op; + + public MaterializePrefixOperation(MaterializeExpression expr, PrefixOperator op) { + this.expr = expr; + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return op.getExpressionType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.getExpectedValue(expectedValue); + } + + public MaterializeDataType[] getInputDataTypes() { + return op.dataTypes; + } + + public String getTextRepresentation() { + return op.textRepresentation; + } + + public MaterializeExpression getExpression() { + return expr; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeSelect.java b/src/sqlancer/materialize/ast/MaterializeSelect.java new file mode 100644 index 000000000..db5b85a2f --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeSelect.java @@ -0,0 +1,145 @@ +package sqlancer.materialize.ast; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public class MaterializeSelect extends SelectBase implements MaterializeExpression, + Select { + + private SelectType selectOption = SelectType.ALL; + private List joinClauses = Collections.emptyList(); + private MaterializeExpression distinctOnClause; + private ForClause forClause; + + public enum ForClause { + UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); + + private final String textRepresentation; + + ForClause(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + public static ForClause getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class MaterializeFromTable implements MaterializeExpression { + private final MaterializeTable t; + private final boolean only; + + public MaterializeFromTable(MaterializeTable t, boolean only) { + this.t = t; + this.only = only; + } + + public MaterializeTable getTable() { + return t; + } + + public boolean isOnly() { + return only; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + } + + public static class MaterializeSubquery implements MaterializeExpression { + private final MaterializeSelect s; + private final String name; + + public MaterializeSubquery(MaterializeSelect s, String name) { + this.s = s; + this.name = name; + } + + public MaterializeSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + } + + public enum SelectType { + DISTINCT, ALL; + + public static SelectType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public void setSelectType(SelectType fromOptions) { + this.setSelectOption(fromOptions); + } + + public void setDistinctOnClause(MaterializeExpression distinctOnClause) { + if (selectOption != SelectType.DISTINCT) { + throw new IllegalArgumentException(); + } + this.distinctOnClause = distinctOnClause; + } + + public SelectType getSelectOption() { + return selectOption; + } + + public void setSelectOption(SelectType fromOptions) { + this.selectOption = fromOptions; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + + @Override + public void setJoinClauses(List joinStatements) { + this.joinClauses = joinStatements; + + } + + @Override + public List getJoinClauses() { + return joinClauses; + } + + public MaterializeExpression getDistinctOnClause() { + return distinctOnClause; + } + + public void setForClause(ForClause forClause) { + this.forClause = forClause; + } + + public ForClause getForClause() { + return forClause; + } + + @Override + public String asString() { + return MaterializeVisitor.asString(this); + } +} diff --git a/src/sqlancer/materialize/ast/MaterializeSimilarTo.java b/src/sqlancer/materialize/ast/MaterializeSimilarTo.java new file mode 100644 index 000000000..dd675050d --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeSimilarTo.java @@ -0,0 +1,40 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeSimilarTo implements MaterializeExpression { + + private final MaterializeExpression string; + private final MaterializeExpression similarTo; + private final MaterializeExpression escapeCharacter; + + public MaterializeSimilarTo(MaterializeExpression string, MaterializeExpression similarTo, + MaterializeExpression escapeCharacter) { + this.string = string; + this.similarTo = similarTo; + this.escapeCharacter = escapeCharacter; + } + + public MaterializeExpression getString() { + return string; + } + + public MaterializeExpression getSimilarTo() { + return similarTo; + } + + public MaterializeExpression getEscapeCharacter() { + return escapeCharacter; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java new file mode 100644 index 000000000..d21c8f81c --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -0,0 +1,416 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public final class MaterializeCommon { + + private MaterializeCommon() { + } + + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); + errors.add("but it cannot be referenced from this part of the query"); + errors.add("missing FROM-clause entry for table"); + + errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in GROUP BY"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + errors.add("result exceeds max size of"); + + errors.add("does not exist"); + errors.add("aggregate functions are not allowed in"); + errors.add("is only defined for finite arguments"); + + return errors; + } + + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("is not commutative"); // exclude + errors.add("operator requires run-time type coercion"); // exclude + + return errors; + } + + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("You might need to add explicit type casts"); + errors.add("invalid regular expression"); + errors.add("could not determine which collation to use"); + errors.add("invalid regular expression"); + errors.add("operator does not exist"); + errors.add("quantifier operand invalid"); + errors.add("collation mismatch"); + errors.add("collations are not supported"); + errors.add("operator is not unique"); + errors.add("is not a valid binary digit"); + errors.add("invalid hexadecimal digit"); + errors.add("invalid hexadecimal data: odd number of digits"); + errors.add("zero raised to a negative power is undefined"); + errors.add("cannot convert infinity to numeric"); + errors.add("division by zero"); + errors.add("invalid input syntax for type money"); + errors.add("invalid input syntax for type"); + errors.add("cannot cast type"); + errors.add("value overflows numeric format"); + errors.add("numeric field overflow"); + errors.add("LIKE pattern must not end with escape character"); + errors.add("is of type boolean but expression is of type text"); + errors.add("a negative number raised to a non-integer power yields a complex result"); + errors.add("could not determine polymorphic type because input has type unknown"); + errors.add("character number must be positive"); + errors.add("unterminated escape sequence"); + errors.add("cannot be matched"); + errors.add("clause must have type"); // "not" in having doesn't work + errors.add("argument must have type"); // "not" in having doesn't work + errors.add("CAST does not support casting from"); + errors.add("aggregate functions are not allowed in"); + errors.add("only defined for finite arguments"); + errors.add("unable to parse column reference in GROUP BY clause"); // TODO + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + private static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("multiple decimal points"); + errors.add("and decimal point together"); + errors.add("multiple decimal points"); + errors.add("cannot use \"S\" twice"); + errors.add("must be ahead of \"PR\""); + errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); + errors.add("cannot use \"S\" and \"SG\" together"); + errors.add("cannot use \"S\" and \"MI\" together"); + errors.add("cannot use \"S\" and \"PL\" together"); + errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); + errors.add("is not a number"); + + return errors; + } + + private static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot XOR bit strings of different sizes"); + errors.add("cannot AND bit strings of different sizes"); + errors.add("cannot OR bit strings of different sizes"); + errors.add("must be type boolean, not type text"); + + return errors; + } + + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("out of valid range"); // get_bit/get_byte + errors.add("cannot take logarithm of a negative number"); + errors.add("cannot take logarithm of zero"); + errors.add("requested character too large for encoding"); // chr + errors.add("null character not permitted"); // chr + errors.add("requested character not valid for encoding"); // chr + errors.add("requested length too large"); // repeat + errors.add("invalid memory alloc request size"); // repeat + errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + + return errors; + } + + private static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("is not a valid hexadecimal digit"); + + return errors; + } + + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("range lower bound must be less than or equal to range upper bound"); + errors.add("result of range difference would not be contiguous"); + errors.add("out of range"); + errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + + return errors; + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getCommonInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("value too long for type character"); + errors.add("not found in view targetlist"); + errors.add("CAST does not support casting from"); + + return errors; + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("unable to parse column reference in GROUP BY clause"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in"); + + return errors; + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); + } + + public static boolean appendDataType(MaterializeDataType type, StringBuilder sb, boolean allowSerial, + boolean generateOnlyKnown, List opClasses) throws AssertionError { + boolean serial = false; + switch (type) { + case BOOLEAN: + sb.append("boolean"); + break; + case INT: + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + break; + case TEXT: + if (Randomly.getBoolean()) { + sb.append("TEXT"); + } else { + if (MaterializeProvider.generateOnlyKnown || Randomly.getBoolean()) { + sb.append("VAR"); + } + sb.append("CHAR"); + sb.append("("); + sb.append(ThreadLocalRandom.current().nextInt(1, 500)); + sb.append(")"); + } + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("REAL"); + break; + case REAL: + sb.append("FLOAT"); + break; + case BIT: + sb.append("INT"); + break; + default: + throw new AssertionError(type); + } + return serial; + } + + public enum TableConstraints { + CHECK, PRIMARY_KEY, FOREIGN_KEY, EXCLUDE + } + + public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder sb, MaterializeTable table, + MaterializeGlobalState globalState, ExpectedErrors errors) { + // TODO constraint name + List tableConstraints = Randomly.nonEmptySubset(TableConstraints.values()); + if (excludePrimaryKey) { + tableConstraints.remove(TableConstraints.PRIMARY_KEY); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + tableConstraints.remove(TableConstraints.FOREIGN_KEY); + } + for (TableConstraints t : tableConstraints) { + sb.append(", "); + // TODO add index parameters + addTableConstraint(sb, table, globalState, t, errors); + } + } + + public static void addTableConstraint(StringBuilder sb, MaterializeTable table, MaterializeGlobalState globalState, + ExpectedErrors errors) { + addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); + } + + private static void addTableConstraint(StringBuilder sb, MaterializeTable table, MaterializeGlobalState globalState, + TableConstraints t, ExpectedErrors errors) { + List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); + List otherColumns; + MaterializeCommon.addCommonExpressionErrors(errors); + switch (t) { + case CHECK: + sb.append("CHECK("); + sb.append(MaterializeVisitor.getExpressionAsString(globalState, MaterializeDataType.BOOLEAN, + table.getColumns())); + sb.append(")"); + errors.add("constraint must be added to child tables too"); + errors.add("missing FROM-clause entry for table"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY("); + sb.append(randomNonEmptyColumnSubset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + break; + case FOREIGN_KEY: + sb.append("FOREIGN KEY ("); + sb.append(randomNonEmptyColumnSubset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(") REFERENCES "); + MaterializeTable randomOtherTable = globalState.getSchema().getRandomTable(tab -> !tab.isView()); + sb.append(randomOtherTable.getName()); + if (randomOtherTable.getColumns().size() < randomNonEmptyColumnSubset.size()) { + throw new IgnoreMeException(); + } + otherColumns = randomOtherTable.getRandomNonEmptyColumnSubset(randomNonEmptyColumnSubset.size()); + sb.append("("); + sb.append(otherColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("MATCH FULL", "MATCH SIMPLE")); + } + if (Randomly.getBoolean()) { + sb.append(" ON DELETE "); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" ON UPDATE "); + errors.add("invalid ON UPDATE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" "); + if (Randomly.getBoolean()) { + sb.append("DEFERRABLE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("INITIALLY DEFERRED", "INITIALLY IMMEDIATE")); + } + } else { + sb.append("NOT DEFERRABLE"); + } + } + break; + case EXCLUDE: + sb.append("EXCLUDE "); + sb.append("("); + // TODO [USING index_method ] + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendExcludeElement(sb, globalState, table.getColumns()); + sb.append(" WITH "); + appendOperator(sb, globalState.getOperators()); + } + sb.append(")"); + errors.add("is not valid"); + errors.add("no operator matches"); + errors.add("operator does not exist"); + errors.add("unknown has no default operator class"); + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("The exclusion operator must be related to the index operator class for the constraint"); + errors.add("could not create exclusion constraint"); + // TODO: index parameters + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append("("); + sb.append(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + table.getColumns(), MaterializeDataType.BOOLEAN))); + sb.append(")"); + } + break; + default: + throw new AssertionError(t); + } + } + + private static void appendOperator(StringBuilder sb, List operators) { + sb.append(Randomly.fromList(operators)); + } + + // complete + private static void appendExcludeElement(StringBuilder sb, MaterializeGlobalState globalState, + List columns) { + if (Randomly.getBoolean()) { + // append column name + sb.append(Randomly.fromList(columns).getName()); + } else { + // append expression + sb.append("("); + sb.append(MaterializeVisitor + .asString(MaterializeExpressionGenerator.generateExpression(globalState, columns))); + sb.append(")"); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromList(globalState.getOpClasses())); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBoolean()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + + private static void deleteOrUpdateAction(StringBuilder sb) { + sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); + } + + public static String getFreeIndexName(MaterializeSchema s) { + List indexNames = s.getIndexNames(); + String candidateName; + do { + candidateName = DBMSCommon.createIndexName((int) Randomly.getNotCachedInteger(0, 100)); + } while (indexNames.contains(candidateName)); + return candidateName; + } +} diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java new file mode 100644 index 000000000..a5483ee1d --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public final class MaterializeDeleteGenerator extends AbstractDeleteGenerator { + + private final MaterializeGlobalState globalState; + + private MaterializeDeleteGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + return new MaterializeDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + MaterializeTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + errors.add("violates foreign key constraint"); + errors.add("violates not-null constraint"); + errors.add("could not determine which collation to use for string comparison"); + appendDeleteFromTable(table.getName()); + if (Randomly.getBoolean()) { + appendWhereClause(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + table.getColumns(), MaterializeDataType.BOOLEAN))); + } + MaterializeCommon.addCommonExpressionErrors(errors); + errors.add("out of range"); + errors.add("does not support casting"); + errors.add("invalid input syntax for"); + errors.add("division by zero"); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java new file mode 100644 index 000000000..4acbd329a --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.materialize.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeIndex; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public final class MaterializeDropIndexGenerator { + + private MaterializeDropIndexGenerator() { + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + MaterializeTable randomTable = globalState.getSchema().getRandomTable(); + List indexes = randomTable.getIndexes(); + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + if (Randomly.getBoolean() || indexes.isEmpty()) { + sb.append("IF EXISTS "); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + if (indexes.isEmpty() || Randomly.getBoolean()) { + sb.append(DBMSCommon.createIndexName(Randomly.smallNumber())); + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", + "cannot drop index", "does not exist"), + true); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java new file mode 100644 index 000000000..f7ff76305 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -0,0 +1,632 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeRowValue; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation.MaterializeBinaryOperator; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation.MaterializeBinaryBitOperator; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConcatOperation; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeFunction.MaterializeFunctionWithResult; +import sqlancer.materialize.ast.MaterializeFunctionWithUnknownResult; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializeOrderByTerm.MaterializeOrder; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression.POSIXRegex; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializePrefixOperation.PrefixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSelect.SelectType; +import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; + +public class MaterializeExpressionGenerator implements ExpressionGenerator, + NoRECGenerator, + TLPWhereGenerator { + + private final int maxDepth; + + private final Randomly r; + + private List columns; + + private List tables; + + private MaterializeRowValue rw; + + private boolean expectedResult; + + private MaterializeGlobalState globalState; + + private boolean allowAggregateFunctions; + + private final Map functionsAndTypes; + + private final List allowedFunctionTypes; + + public MaterializeExpressionGenerator(MaterializeGlobalState globalState) { + this.r = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + this.globalState = globalState; + this.functionsAndTypes = globalState.getFunctionsAndTypes(); + this.allowedFunctionTypes = globalState.getAllowedFunctionTypes(); + } + + public MaterializeExpressionGenerator setColumns(List columns) { + this.columns = columns; + return this; + } + + public MaterializeExpressionGenerator setRowValue(MaterializeRowValue rw) { + this.rw = rw; + return this; + } + + public MaterializeExpression generateExpression(int depth) { + return generateExpression(depth, MaterializeDataType.getRandomType()); + } + + @Override + public List generateOrderBys() { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new MaterializeOrderByTerm(MaterializeColumnValue.create(Randomly.fromList(columns), null), + MaterializeOrder.getRandomOrder())); + } + return orderBys; + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, LIKE, BETWEEN, IN_OPERATION, + POSIX_REGEX; + } + + private MaterializeExpression generateFunctionWithUnknownResult(int depth, MaterializeDataType type) { + List supportedFunctions = MaterializeFunctionWithUnknownResult + .getSupportedFunctions(type); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + supportedFunctions = supportedFunctions.stream() + .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (supportedFunctions.isEmpty()) { + throw new IgnoreMeException(); + } + MaterializeFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); + return new MaterializeFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); + } + + private MaterializeExpression generateFunctionWithKnownResult(int depth, MaterializeDataType type) { + List functions = Stream + .of(MaterializeFunction.MaterializeFunctionWithResult.values()).filter(f -> f.supportsReturnType(type)) + .collect(Collectors.toList()); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (functions.isEmpty()) { + throw new IgnoreMeException(); + } + MaterializeFunctionWithResult randomFunction = Randomly.fromList(functions); + int nrArgs = randomFunction.getNrArgs(); + if (randomFunction.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + MaterializeDataType[] argTypes = randomFunction.getInputTypesForReturnType(type, nrArgs); + MaterializeExpression[] args = new MaterializeExpression[nrArgs]; + do { + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1, argTypes[i]); + } + } while (!randomFunction.checkArguments(args)); + return new MaterializeFunction(randomFunction, type, args); + } + + private MaterializeExpression generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (MaterializeProvider.generateOnlyKnown) { + validOptions.remove(BooleanExpression.POSIX_REGEX); + } + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + PostfixOperator random = PostfixOperator.getRandom(); + return MaterializePostfixOperation + .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); + case IN_OPERATION: + return inOperation(depth + 1); + case NOT: + return new MaterializePrefixOperation(generateExpression(depth + 1, MaterializeDataType.BOOLEAN), + PrefixOperator.NOT); + case BINARY_LOGICAL_OPERATOR: + MaterializeExpression first = generateExpression(depth + 1, MaterializeDataType.BOOLEAN); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + first = new MaterializeBinaryLogicalOperation(first, + generateExpression(depth + 1, MaterializeDataType.BOOLEAN), BinaryLogicalOperator.getRandom()); + } + return first; + case BINARY_COMPARISON: + MaterializeDataType dataType = getMeaningfulType(); + return generateComparison(depth, dataType); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.BOOLEAN); + case LIKE: + return new MaterializeLikeOperation(generateExpression(depth + 1, MaterializeDataType.TEXT), + generateExpression(depth + 1, MaterializeDataType.TEXT)); + case BETWEEN: + MaterializeDataType type = getMeaningfulType(); + return new MaterializeBetweenOperation(generateExpression(depth + 1, type), + generateExpression(depth + 1, type), generateExpression(depth + 1, type), Randomly.getBoolean()); + case POSIX_REGEX: + assert !expectedResult; + return new MaterializePOSIXRegularExpression(generateExpression(depth + 1, MaterializeDataType.TEXT), + generateExpression(depth + 1, MaterializeDataType.TEXT), POSIXRegex.getRandom()); + default: + throw new AssertionError(); + } + } + + private MaterializeDataType getMeaningfulType() { + // make it more likely that the expression does not only consist of constant + // expressions + if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { + return MaterializeDataType.getRandomType(); + } else { + return Randomly.fromList(columns).getType(); + } + } + + private MaterializeExpression generateFunction(int depth, MaterializeDataType type) { + if (MaterializeProvider.generateOnlyKnown || Randomly.getBoolean()) { + return generateFunctionWithKnownResult(depth, type); + } else { + return generateFunctionWithUnknownResult(depth, type); + } + } + + private MaterializeExpression generateComparison(int depth, MaterializeDataType dataType) { + MaterializeExpression leftExpr = generateExpression(depth + 1, dataType); + MaterializeExpression rightExpr = generateExpression(depth + 1, dataType); + return getComparison(leftExpr, rightExpr); + } + + private MaterializeExpression getComparison(MaterializeExpression leftExpr, MaterializeExpression rightExpr) { + return new MaterializeBinaryComparisonOperation(leftExpr, rightExpr, + MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator.getRandom()); + } + + private MaterializeExpression inOperation(int depth) { + MaterializeDataType type = MaterializeDataType.getRandomType(); + MaterializeExpression leftExpr = generateExpression(depth + 1, type); + List rightExpr = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + rightExpr.add(generateExpression(depth + 1, type)); + } + return new MaterializeInOperation(leftExpr, rightExpr, Randomly.getBoolean()); + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + MaterializeDataType type) { + return new MaterializeExpressionGenerator(globalState).generateExpression(0, type); + } + + public MaterializeExpression generateExpression(int depth, MaterializeDataType originalType) { + MaterializeDataType dataType = originalType; + if (dataType == MaterializeDataType.REAL && Randomly.getBoolean()) { + dataType = Randomly.fromOptions(MaterializeDataType.INT, MaterializeDataType.FLOAT); + } + if (dataType == MaterializeDataType.FLOAT && Randomly.getBoolean()) { + dataType = MaterializeDataType.INT; + } + if (!filterColumns(dataType).isEmpty() && Randomly.getBoolean()) { + return createColumnOfType(dataType); + } + return generateExpressionInternal(depth, dataType); + } + + private MaterializeExpression generateExpressionInternal(int depth, MaterializeDataType dataType) + throws AssertionError { + if (allowAggregateFunctions && Randomly.getBoolean()) { + allowAggregateFunctions = false; // aggregate function calls cannot be nested + return getAggregate(dataType); + } + if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { + // generic expression + if (Randomly.getBoolean() || depth > maxDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(r, dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(r, dataType); + } else { + return createColumnOfType(dataType); + } + } + } else { + if (Randomly.getBoolean()) { + return new MaterializeCastOperation(generateExpression(depth + 1), getCompoundDataType(dataType)); + } else { + return generateFunctionWithUnknownResult(depth, dataType); + } + } + } else { + switch (dataType) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case TEXT: + return generateTextExpression(depth); + case DECIMAL: + case REAL: + case FLOAT: + return generateConstant(r, dataType); + case BIT: + return generateBitExpression(depth); + default: + throw new AssertionError(dataType); + } + } + } + + private static MaterializeCompoundDataType getCompoundDataType(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + case DECIMAL: // TODO + case FLOAT: + case INT: + case REAL: + case BIT: + return MaterializeCompoundDataType.create(type); + case TEXT: // TODO + if (Randomly.getBoolean() || MaterializeProvider.generateOnlyKnown /* + * The PQS implementation does not check + * for size specifications + */) { + return MaterializeCompoundDataType.create(type); + } else { + return MaterializeCompoundDataType.create(type, (int) Randomly.getNotCachedInteger(1, 1000)); + } + default: + throw new AssertionError(type); + } + + } + + private enum TextExpression { + CAST, FUNCTION, CONCAT + } + + private MaterializeExpression generateTextExpression(int depth) { + TextExpression option; + List validOptions = new ArrayList<>(Arrays.asList(TextExpression.values())); + option = Randomly.fromList(validOptions); + + switch (option) { + case CAST: + return new MaterializeCastOperation(generateExpression(depth + 1), + getCompoundDataType(MaterializeDataType.TEXT)); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.TEXT); + case CONCAT: + return generateConcat(depth); + default: + throw new AssertionError(); + } + } + + private MaterializeExpression generateConcat(int depth) { + MaterializeExpression left = generateExpression(depth + 1, MaterializeDataType.TEXT); + MaterializeExpression right = generateExpression(depth + 1); + return new MaterializeConcatOperation(left, right); + } + + private enum BitExpression { + BINARY_OPERATION + }; + + private MaterializeExpression generateBitExpression(int depth) { + BitExpression option; + option = Randomly.fromOptions(BitExpression.values()); + switch (option) { + case BINARY_OPERATION: + return new MaterializeBinaryBitOperation(MaterializeBinaryBitOperator.getRandom(), + generateExpression(depth + 1, MaterializeDataType.BIT), + generateExpression(depth + 1, MaterializeDataType.BIT)); + default: + throw new AssertionError(); + } + } + + private enum IntExpression { + UNARY_OPERATION, FUNCTION, /* CAST, */BINARY_ARITHMETIC_EXPRESSION + } + + private MaterializeExpression generateIntExpression(int depth) { + IntExpression option; + option = Randomly.fromOptions(IntExpression.values()); + switch (option) { + case UNARY_OPERATION: + MaterializeExpression intExpression = generateExpression(depth + 1, MaterializeDataType.INT); + return new MaterializePrefixOperation(intExpression, + Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.INT); + case BINARY_ARITHMETIC_EXPRESSION: + return new MaterializeBinaryArithmeticOperation(generateExpression(depth + 1, MaterializeDataType.INT), + generateExpression(depth + 1, MaterializeDataType.INT), MaterializeBinaryOperator.getRandom()); + default: + throw new AssertionError(); + } + } + + private MaterializeExpression createColumnOfType(MaterializeDataType type) { + List columns = filterColumns(type); + MaterializeColumn fromList = Randomly.fromList(columns); + MaterializeConstant value = rw == null ? null : rw.getValues().get(fromList); + return MaterializeColumnValue.create(fromList, value); + } + + final List filterColumns(MaterializeDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + public MaterializeExpression generateExpressionWithExpectedResult(MaterializeDataType type) { + this.expectedResult = true; + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns) + .setRowValue(rw); + MaterializeExpression expr; + do { + expr = gen.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + public static MaterializeExpression generateConstant(Randomly r, MaterializeDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return MaterializeConstant.createNullConstant(); + } + switch (type) { + case INT: + if (Randomly.getBooleanWithSmallProbability()) { + return MaterializeConstant.createTextConstant(String.valueOf(r.getInteger())); + } else { + return MaterializeConstant.createIntConstant(r.getInteger()); + } + case BOOLEAN: + if (Randomly.getBooleanWithSmallProbability() && !MaterializeProvider.generateOnlyKnown) { + return MaterializeConstant + .createTextConstant(Randomly.fromOptions("TR", "TRUE", "FA", "FALSE", "0", "1", "ON", "off")); + } else { + return MaterializeConstant.createBooleanConstant(Randomly.getBoolean()); + } + case TEXT: + return MaterializeConstant.createTextConstant(r.getString()); + case DECIMAL: + return MaterializeConstant.createDecimalConstant(r.getRandomBigDecimal()); + case FLOAT: + return MaterializeConstant.createFloatConstant((float) r.getDouble()); + case REAL: + return MaterializeConstant.createDoubleConstant(r.getDouble()); + case BIT: + return MaterializeConstant.createBitConstant(r.getInteger()); + default: + throw new AssertionError(type); + } + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + List columns, MaterializeDataType type) { + return new MaterializeExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + List columns) { + return new MaterializeExpressionGenerator(globalState).setColumns(columns).generateExpression(0); + + } + + public List generateExpressions(int nr) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(0)); + } + return expressions; + } + + public MaterializeExpression generateExpression(MaterializeDataType dataType) { + return generateExpression(0, dataType); + } + + public MaterializeExpressionGenerator setGlobalState(MaterializeGlobalState globalState) { + this.globalState = globalState; + return this; + } + + public MaterializeExpression generateHavingClause() { + this.allowAggregateFunctions = true; + MaterializeExpression expression = generateExpression(MaterializeDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + + public MaterializeExpression generateAggregate() { + return getAggregate(MaterializeDataType.getRandomType()); + } + + private MaterializeExpression getAggregate(MaterializeDataType dataType) { + List aggregates = MaterializeAggregateFunction.getAggregates(dataType); + MaterializeAggregateFunction agg = Randomly.fromList(aggregates); + return generateArgsForAggregate(dataType, agg); + } + + public MaterializeAggregate generateArgsForAggregate(MaterializeDataType dataType, + MaterializeAggregateFunction agg) { + List types = agg.getTypes(dataType); + List args = new ArrayList<>(); + for (MaterializeDataType argType : types) { + args.add(generateExpression(argType)); + } + return new MaterializeAggregate(args, agg); + } + + public MaterializeExpressionGenerator allowAggregates(boolean value) { + allowAggregateFunctions = value; + return this; + } + + @Override + public MaterializeExpression generatePredicate() { + return generateExpression(MaterializeDataType.BOOLEAN); + } + + @Override + public MaterializeExpression negatePredicate(MaterializeExpression predicate) { + return new MaterializePrefixOperation(predicate, MaterializePrefixOperation.PrefixOperator.NOT); + } + + @Override + public MaterializeExpression isNull(MaterializeExpression expr) { + return new MaterializePostfixOperation(expr, PostfixOperator.IS_NULL); + } + + @Override + public MaterializeExpressionGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public MaterializeExpression generateBooleanExpression() { + return generateExpression(MaterializeDataType.BOOLEAN); + } + + @Override + public MaterializeSelect generateSelect() { + return new MaterializeSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeTable table = Randomly.fromList(tables); + tables.remove(table); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, + options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); + joinStatements.add(j); + } + + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(MaterializeSelect select, MaterializeExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + MaterializeAggregate aggr = new MaterializeAggregate( + List.of(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)), + MaterializeAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(List.of(allColumns)); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + select.setSelectType(SelectType.ALL); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(MaterializeSelect select, MaterializeExpression whereCondition) { + MaterializeCastOperation isTrue = new MaterializeCastOperation(whereCondition, + MaterializeCompoundDataType.create(MaterializeDataType.INT)); + MaterializePostfixText asText = new MaterializePostfixText(isTrue, " as count", null, MaterializeDataType.INT); + select.setFetchColumns(List.of(asText)); + select.setSelectType(SelectType.ALL); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(columns); + for (MaterializeColumn c : targetColumns) { + fetchColumns.add(new MaterializeColumnValue(c, null)); + } + return fetchColumns; + } +} diff --git a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java new file mode 100644 index 000000000..a63353f1f --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java @@ -0,0 +1,84 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public class MaterializeIndexGenerator extends AbstractIndexGenerator { + + private final MaterializeGlobalState globalState; + + public MaterializeIndexGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; + } + + public enum IndexType { + BTREE, HASH, GIST, GIN + } + + public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { + return new MaterializeIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(false); + MaterializeTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized + // views + sb.append(MaterializeCommon.getFreeIndexName(globalState.getSchema())); + sb.append(" ON "); + sb.append(randomTable.getName()); + IndexType method = IndexType.BTREE; + + sb.append("("); + if (method == IndexType.HASH) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(randomTable.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + } + + sb.append(")"); + errors.add("already contains data"); // CONCURRENT INDEX failed + errors.add("You might need to add explicit type casts"); + errors.add(" collations are not supported"); + errors.add("because it has pending trigger events"); + errors.add("could not determine which collation to use for index expression"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("is duplicated"); + errors.add("access method \"gin\" does not support unique indexes"); + errors.add("access method \"hash\" does not support unique indexes"); + errors.add("already exists"); + errors.add("could not create unique index"); + errors.add("has no default operator class"); + errors.add("does not support"); + errors.add("does not support casting"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("invalid input syntax for"); + errors.add("must be type "); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("out of range"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("result of range difference would not be contiguous"); + errors.add("which is part of the partition key"); + MaterializeCommon.addCommonExpressionErrors(errors); + } +} diff --git a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java new file mode 100644 index 000000000..7a5374b95 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java @@ -0,0 +1,106 @@ +package sqlancer.materialize.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; + +public final class MaterializeInsertGenerator { + + private MaterializeInsertGenerator() { + } + + public static SQLQueryAdapter insert(MaterializeGlobalState globalState) { + MaterializeTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("cannot insert into column"); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonInsertUpdateErrors(errors); + MaterializeCommon.addCommonExpressionErrors(errors); + errors.add("multiple assignments to same column"); + errors.add("violates foreign key constraint"); + errors.add("value too long for type character"); + errors.add("conflicting key value violates exclusion constraint"); + errors.add("violates not-null constraint"); + errors.add("current transaction is aborted"); + errors.add("bit string too long"); + errors.add("new row violates check option for view"); + errors.add("reached maximum value of sequence"); + errors.add("but expression is of type"); + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES"); + + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + StringBuilder sbRowValue = new StringBuilder(); + sbRowValue.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sbRowValue.append(", "); + } + sbRowValue.append(MaterializeVisitor.asString(MaterializeExpressionGenerator + .generateConstant(globalState.getRandomly(), columns.get(i).getType()))); + } + sbRowValue.append(")"); + + int n = (int) Randomly.getNotCachedInteger(100, 1000); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(sbRowValue); + } + } else { + int n = Randomly.smallNumber() + 1; + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + insertRow(globalState, sb, columns); + } + } + errors.add("duplicate key value violates unique constraint"); + errors.add("identity column defined as GENERATED ALWAYS"); + errors.add("out of range"); + errors.add("violates check constraint"); + errors.add("no partition of relation"); + errors.add("invalid input syntax"); + errors.add("division by zero"); + errors.add("violates foreign key constraint"); + errors.add("data type unknown"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private static void insertRow(MaterializeGlobalState globalState, StringBuilder sb, + List columns) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + MaterializeExpression generateConstant; + if (Randomly.getBoolean()) { + generateConstant = MaterializeExpressionGenerator.generateConstant(globalState.getRandomly(), + columns.get(i).getType()); + } else { + generateConstant = new MaterializeExpressionGenerator(globalState) + .generateExpression(columns.get(i).getType()); + } + sb.append(MaterializeVisitor.asString(generateConstant)); + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java new file mode 100644 index 000000000..05d908e05 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java @@ -0,0 +1,64 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.ForClause; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.SelectType; + +public final class MaterializeRandomQueryGenerator { + + private MaterializeRandomQueryGenerator() { + } + + public static MaterializeSelect createRandomQuery(int nrColumns, MaterializeGlobalState globalState) { + List columns = new ArrayList<>(); + MaterializeTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState) + .setColumns(tables.getColumns()); + for (int i = 0; i < nrColumns; i++) { + columns.add(gen.generateExpression(0)); + } + MaterializeSelect select = new MaterializeSelect(); + select.setSelectType(SelectType.getRandom()); + if (select.getSelectOption() == SelectType.DISTINCT && Randomly.getBoolean()) { + select.setDistinctOnClause(gen.generateExpression(0)); + } + select.setFromList(tables.getTables().stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return select; + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java new file mode 100644 index 000000000..c6772db47 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java @@ -0,0 +1,128 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public class MaterializeTableGenerator { + + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + @SuppressWarnings("unused") + private boolean isTemporaryTable; + @SuppressWarnings("unused") + private final MaterializeSchema newSchema; + private final List columnsToBeAdded = new ArrayList<>(); + protected final ExpectedErrors errors = new ExpectedErrors(); + private final MaterializeTable table; + private final boolean generateOnlyKnown; + private final MaterializeGlobalState globalState; + + public MaterializeTableGenerator(String tableName, MaterializeSchema newSchema, boolean generateOnlyKnown, + MaterializeGlobalState globalState) { + this.tableName = tableName; + this.newSchema = newSchema; + this.generateOnlyKnown = generateOnlyKnown; + this.globalState = globalState; + table = new MaterializeTable(tableName, columnsToBeAdded, null, null, null, false, false); + errors.add("invalid input syntax for"); + errors.add("is not unique"); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("cannot create partitioned table as inheritance child"); + errors.add("does not support casting"); + errors.add("ERROR: functions in index expression must be marked IMMUTABLE"); + errors.add("functions in partition key expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not exist for access method"); + errors.add("does not accept data type"); + errors.add("but default expression is of type text"); + errors.add("has pseudo-type unknown"); + errors.add("no collation was derived for partition key column"); + errors.add("inherits from generated column but specifies identity"); + errors.add("inherits from generated column but specifies default"); + errors.add("already exists"); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonTableErrors(errors); + } + + public static SQLQueryAdapter generate(String tableName, MaterializeSchema newSchema, boolean generateOnlyKnown, + MaterializeGlobalState globalState) { + return new MaterializeTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); + } + + protected SQLQueryAdapter generate() { + sb.append("CREATE"); + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + createStandard(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void createStandard() throws AssertionError { + sb.append("("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + String name = DBMSCommon.createColumnName(i); + createColumn(name); + } + sb.append(")"); + } + + private void createColumn(String name) throws AssertionError { + sb.append(name); + sb.append(" "); + MaterializeDataType type = MaterializeDataType.getRandomType(); + MaterializeCommon.appendDataType(type, sb, true, generateOnlyKnown, globalState.getCollates()); + MaterializeColumn c = new MaterializeColumn(name, type); + c.setTable(table); + columnsToBeAdded.add(c); + sb.append(" "); + if (Randomly.getBoolean()) { + createColumnConstraint(type); + } + } + + private enum ColumnConstraint { + DEFAULT + }; + + private void createColumnConstraint(MaterializeDataType type) { + List constraintSubset = Randomly.nonEmptySubset(ColumnConstraint.values()); + for (ColumnConstraint c : constraintSubset) { + sb.append(" "); + switch (c) { + case DEFAULT: + sb.append("DEFAULT"); + sb.append(" ("); + sb.append(MaterializeVisitor + .asString(MaterializeExpressionGenerator.generateExpression(globalState, type))); + sb.append(")"); + // CREATE TEMPORARY TABLE t1(c0 smallint DEFAULT ('566963878')); + errors.add("out of range"); + errors.add("is a generated column"); + break; + default: + throw new AssertionError(sb); + } + } + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java new file mode 100644 index 000000000..abd288f99 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -0,0 +1,77 @@ +package sqlancer.materialize.gen; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; + +public final class MaterializeUpdateGenerator extends AbstractUpdateGenerator { + + private final MaterializeGlobalState globalState; + private MaterializeTable randomTable; + + private MaterializeUpdateGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "does not support casting", "must be type boolean", + "is not unique", " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable")); + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + return new MaterializeUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + sb.append("UPDATE "); + sb.append(randomTable.getName()); + sb.append(" SET "); + errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced + // table multiple times + errors.add("new row violates check option for view"); + MaterializeCommon.addCommonInsertUpdateErrors(errors); + updateColumns(columns); + errors.add("invalid input syntax for "); + errors.add("operator does not exist: text = boolean"); + errors.add("violates check constraint"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("but expression is of type"); + MaterializeCommon.addCommonExpressionErrors(errors); + if (!Randomly.getBooleanWithSmallProbability()) { + MaterializeExpression where = MaterializeExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), MaterializeDataType.BOOLEAN); + appendWhereClause(MaterializeVisitor.asString(where)); + } + } + + @Override + protected void updateValue(MaterializeColumn column) { + if (!Randomly.getBoolean()) { + MaterializeExpression constant = MaterializeExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(MaterializeVisitor.asString(constant)); + } else { + sb.append("("); + MaterializeExpression expr = MaterializeExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), column.getType()); + // caused by casts + sb.append(MaterializeVisitor.asString(expr)); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java new file mode 100644 index 000000000..e3cb8ff29 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java @@ -0,0 +1,65 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeSelect; + +public final class MaterializeViewGenerator { + + private MaterializeViewGenerator() { + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + @SuppressWarnings("unused") + boolean materialized; + @SuppressWarnings("unused") + boolean recursive = false; + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + materialized = true; + } else { + if (Randomly.getBoolean()) { + sb.append(" OR REPLACE"); + } + materialized = false; + } + sb.append(" VIEW "); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); + sb.append("("); + int nrColumns = Randomly.smallNumber() + 1; + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(DBMSCommon.createColumnName(i)); + } + sb.append(")"); + sb.append(" AS ("); + MaterializeSelect select = MaterializeRandomQueryGenerator.createRandomQuery(nrColumns, globalState); + sb.append(MaterializeVisitor.asString(select)); + sb.append(")"); + MaterializeCommon.addGroupingErrors(errors); + errors.add("already exists"); + errors.add("cannot drop columns from view"); + errors.add("non-integer constant in ORDER BY"); // TODO + errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO + errors.add("cannot change data type of view column"); + errors.add("specified more than once"); // TODO + errors.add("materialized views must not use temporary tables or views"); + errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); + errors.add("is not a view"); + errors.add("non-integer constant in DISTINCT ON"); + errors.add("unable to parse column reference in DISTINCT ON clause"); + errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + MaterializeCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..50dff2b49 --- /dev/null +++ b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java @@ -0,0 +1,151 @@ +package sqlancer.materialize.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeRowValue; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public class MaterializePivotedQuerySynthesisOracle extends + PivotedQuerySynthesisBase { + + private List fetchColumns; + + public MaterializePivotedQuerySynthesisOracle(MaterializeGlobalState globalState) throws SQLException { + super(globalState); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonFetchErrors(errors); + } + + @Override + public SQLQueryAdapter getRectifiedQuery() throws SQLException { + MaterializeTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + + MaterializeSelect selectStatement = new MaterializeSelect(); + selectStatement.setSelectType(Randomly.fromOptions(MaterializeSelect.SelectType.values())); + List columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + fetchColumns = columns; + selectStatement.setFromList(randomFromTables.getTables().stream().map(t -> new MaterializeFromTable(t, false)) + .collect(Collectors.toList())); + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new MaterializeColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + MaterializeExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + MaterializeExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + MaterializeExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = new MaterializeExpressionGenerator(globalState).setColumns(columns) + .generateOrderBys(); + selectStatement.setOrderByClauses(orderBy); + return new SQLQueryAdapter(MaterializeVisitor.asString(selectStatement)); + } + + /* + * Prevent name collisions by aliasing the column. + */ + private MaterializeColumn getFetchValueAliasedColumn(MaterializeColumn c) { + MaterializeColumn aliasedColumn = new MaterializeColumn( + c.getName() + " AS " + c.getTable().getName() + c.getName(), c.getType()); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + private List generateGroupByClause(List columns, MaterializeRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> MaterializeColumnValue.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private MaterializeConstant generateLimit() { + if (Randomly.getBoolean()) { + return MaterializeConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private MaterializeExpression generateOffset() { + if (Randomly.getBoolean()) { + return MaterializeConstant.createIntConstant(0); + } else { + return null; + } + } + + private MaterializeExpression generateRectifiedExpression(List columns, MaterializeRowValue rw) { + MaterializeExpression expr = new MaterializeExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(MaterializeDataType.BOOLEAN); + MaterializeExpression result; + if (expr.getExpectedValue().isNull()) { + result = MaterializePostfixOperation.create(expr, PostfixOperator.IS_NULL); + } else { + result = MaterializePostfixOperation.create(expr, + expr.getExpectedValue().cast(MaterializeDataType.BOOLEAN).asBoolean() ? PostfixOperator.IS_TRUE + : PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (MaterializeColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + @Override + protected String getExpectedValues(MaterializeExpression expr) { + return MaterializeVisitor.asExpectedValues(expr); + } + +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java new file mode 100644 index 000000000..6bbd7d795 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java @@ -0,0 +1,187 @@ +package sqlancer.materialize.oracle.tlp; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.postgresql.util.PSQLException; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; +import sqlancer.materialize.ast.MaterializeAlias; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializePrefixOperation.PrefixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.gen.MaterializeCommon; + +public class MaterializeTLPAggregateOracle extends MaterializeTLPBase implements TestOracle { + private String generatedQueryString; + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public MaterializeTLPAggregateOracle(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() throws SQLException { + MaterializeAggregateFunction aggregateFunction = Randomly.fromOptions(MaterializeAggregateFunction.MAX, + MaterializeAggregateFunction.MIN, MaterializeAggregateFunction.SUM, + MaterializeAggregateFunction.BIT_AND, MaterializeAggregateFunction.BIT_OR, + MaterializeAggregateFunction.BOOL_AND, MaterializeAggregateFunction.BOOL_OR, + MaterializeAggregateFunction.COUNT); + MaterializeAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), + aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + originalQuery = MaterializeVisitor.asString(select); + generatedQueryString = originalQuery; + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, + secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + private String createMetamorphicUnionQuery(MaterializeSelect select, MaterializeAggregate aggregate, + List from) { + String metamorphicQuery; + MaterializeExpression whereClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeExpression negatedClause = new MaterializePrefixOperation(whereClause, PrefixOperator.NOT); + MaterializeExpression notNullClause = new MaterializePostfixOperation(whereClause, PostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + MaterializeSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); + MaterializeSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); + MaterializeSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += MaterializeVisitor.asString(leftSelect) + " UNION ALL " + + MaterializeVisitor.asString(middleSelect) + " UNION ALL " + MaterializeVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + } catch (PSQLException e) { + throw new AssertionError(queryString, e); + } + return resultString; + } + + private List mapped(MaterializeAggregate aggregate) { + switch (aggregate.getFunction()) { + case SUM: + case COUNT: + case BIT_AND: + case BIT_OR: + case BOOL_AND: + case BOOL_OR: + case MAX: + case MIN: + return aliasArgs(Arrays.asList(aggregate)); + default: + throw new AssertionError(aggregate.getFunction()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (MaterializeExpression expr : originalAggregateArgs) { + args.add(new MaterializeAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(MaterializeAggregate aggregate) { + switch (aggregate.getFunction()) { + case COUNT: + return MaterializeAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunction().toString() + "(agg0)"; + } + } + + private MaterializeSelect getSelect(List aggregates, List from, + MaterializeExpression whereClause, List joinList) { + MaterializeSelect leftSelect = new MaterializeSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinClauses(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java new file mode 100644 index 000000000..faf136c21 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -0,0 +1,144 @@ +package sqlancer.materialize.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.ForClause; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public class MaterializeTLPBase + extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + protected MaterializeSchema s; + protected MaterializeTables targetTables; + protected MaterializeExpressionGenerator gen; + protected MaterializeSelect select; + + public MaterializeTLPBase(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonFetchErrors(errors); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(state, targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(MaterializeGlobalState globalState, + List columns, List tables) { + List joinStatements = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeTable table = Randomly.fromList(tables); + tables.remove(table); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, + options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); + joinStatements.add(j); + } + + return joinStatements; + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream() + .map(t -> new MaterializeFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + gen = new MaterializeExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new MaterializeSelect(); + select.setFetchColumns(generateFetchColumns()); + select.setFromList(tableList); + select.setWhereClause(null); + select.setJoinClauses(joins); + if (Randomly.getBoolean()) { + select.setForClause(ForClause.getRandom()); + } + } + + List generateFetchColumns() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(targetTables.getColumns()); + for (MaterializeColumn c : targetColumns) { + fetchColumns.add(new MaterializeColumnValue(c, null)); + } + return fetchColumns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + + public static MaterializeSubquery createSubquery(MaterializeGlobalState globalState, String name, + MaterializeTables tables) { + List columns = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState) + .setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + MaterializeSelect select = new MaterializeSelect(); + select.setFromList(tables.getTables().stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return new MaterializeSubquery(select, name); + } +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java new file mode 100644 index 000000000..deefb4fb9 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java @@ -0,0 +1,84 @@ +package sqlancer.materialize.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.gen.MaterializeCommon; + +public class MaterializeTLPHavingOracle extends MaterializeTLPBase { + private String generatedQueryString; + + public MaterializeTLPHavingOracle(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + havingCheck(); + } + + protected void havingCheck() throws SQLException { + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(MaterializeDataType.BOOLEAN)); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = MaterializeVisitor.asString(select); + generatedQueryString = originalQueryString; + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + // See https://github.com/MaterializeInc/materialize/issues/18346, have to check + // if predicate errors by putting + // it in SELECT first + List originalColumns = select.getFetchColumns(); + List checkColumns = new ArrayList<>(); + checkColumns.add(predicate); + select.setFetchColumns(checkColumns); + String errorCheckQueryString = MaterializeVisitor.asString(select); + ComparatorHelper.getResultSetFirstColumnAsString(errorCheckQueryString, errors, state); + select.setFetchColumns(originalColumns); + + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + select.setHavingClause(predicate); + String firstQueryString = MaterializeVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = MaterializeVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = MaterializeVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + protected MaterializeExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List generateFetchColumns() { + List expressions = gen.allowAggregates(true) + .generateExpressions(Randomly.smallNumber() + 1); + gen.allowAggregates(false); + return expressions; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 9cd87614d..e4fae3cd7 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -3,12 +3,8 @@ // do not make the fields final to avoid warnings public final class MySQLBugs { - // https://bugs.mysql.com/bug.php?id=99127 0.9 > t0.c0 malfunctions when c0 is - // an INT UNSIGNED - public static boolean bug99127 = true; - // https://bugs.mysql.com/99182 BETWEEN malfunctions for DECIMAL and TEXT - public static boolean bug99181 = true; + public static boolean bug99182 = true; // https://bugs.mysql.com/bug.php?id=99183 public static boolean bug99183 = true; @@ -19,6 +15,37 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=99135 public static boolean bug99135 = true; + // https://bugs.mysql.com/bug.php?id=111471 + public static boolean bug111471 = true; + + // https://bugs.mysql.com/bug.php?id=112242 + public static boolean bug112242 = true; + + // https://bugs.mysql.com/bug.php?id=112243 + public static boolean bug112243 = true; + + // https://bugs.mysql.com/bug.php?id=112264 + public static boolean bug112264 = true; + + // https://bugs.mysql.com/bug.php?id=114533 + public static boolean bug114533 = true; + + // https://bugs.mysql.com/bug.php?id=114534 + public static boolean bug114534 = true; + + // https://bugs.mysql.com/bug.php?id=120710 + // Inserting a NULL and a value which rounds to 0 into a DECIMAL column causes result set mismatch. + public static boolean bug120710 = true; + + // https://bugs.mysql.com/bug.php?id=120711 + // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set + // mismatch. + public static boolean bug120711 = true; + + // https://bugs.mysql.com/bug.php?id=120712 + // Creating an index in between two NULL inserts causes inconsistent CERT result. + public static boolean bug120712 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index c2893f72f..0b6eb8284 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -1,15 +1,85 @@ package sqlancer.mysql; -import java.util.Set; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import sqlancer.common.query.ExpectedErrors; public final class MySQLErrors { private MySQLErrors() { } - public static void addExpressionErrors(Set errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) errors.add("is not valid for CHARACTER SET"); + + if (MySQLBugs.bug111471) { + errors.add("Memory capacity exceeded"); + } + + return errors; + } + + public static List getExpressionRegexErrors() { + ArrayList errors = new ArrayList<>(); + + if (MySQLBugs.bug114533) { + errors.add(Pattern.compile("For input string: \"0+-0\"")); // match: For input string: + // "00000000000000000000-0" + } + + errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); + errors.add(Pattern.compile("Unknown column '.*' in 'EXISTS subquery'")); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + errors.addAllRegexes(getExpressionRegexErrors()); + } + + public static List getInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("doesn't have a default value"); + errors.add("Data truncation"); + errors.add("Incorrect integer value"); + errors.add("Incorrect FLOAT value"); + errors.add("Incorrect DOUBLE value"); + errors.add("Duplicate entry"); + errors.add("Data truncated for column"); + errors.add("Data truncated for functional index"); + errors.add("cannot be null"); + errors.add("Incorrect decimal value"); + errors.add("The value specified for generated column"); + + return errors; + } + + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getInsertUpdateErrors()); + } + + public static List getDMLErrors() { + ArrayList errors = new ArrayList<>(getInsertUpdateErrors()); + + // WHERE-clause type coercion (e.g. string -> number) is only a warning in SELECT but a hard error in + // DELETE/UPDATE under strict sql_mode (MySQL 1292). A semantics-preserving transform may benignly change + // whether it fires, so it is tolerated rather than flagged. + errors.add("Truncated incorrect"); + // Foreign key constraint failure when deleting/updating a referenced row. + errors.add("a foreign key constraint fails"); + + return errors; + } + + public static void addDMLErrors(ExpectedErrors errors) { + errors.addAll(getDMLErrors()); } } diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index 437c7b654..8ad2d8b2b 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -1,10 +1,14 @@ package sqlancer.mysql; +import java.util.List; + import sqlancer.IgnoreMeException; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -13,10 +17,12 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; public class MySQLExpectedValueVisitor implements MySQLVisitor { @@ -31,7 +37,8 @@ private void print(MySQLExpression expr) { sb.append("\t"); } sb.append(v.get()); - sb.append(" -- " + expr.getExpectedValue()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); sb.append("\n"); } @@ -152,4 +159,47 @@ public void visit(MySQLCollate collate) { visit(collate.getExpectedValue()); } + @Override + public void visit(MySQLJoin join) { + print(join); + visit(join.getOnClause()); + } + + @Override + public void visit(MySQLText text) { + print(text); + } + + @Override + public void visit(MySQLAggregate aggr) { + // PQS is currently unsupported for aggregates. + throw new IgnoreMeException(); + } + + @Override + public void visit(MySQLCaseOperator caseOp) { + print(caseOp); + + MySQLExpression switchCondition = caseOp.getSwitchCondition(); + if (switchCondition != null) { + print(switchCondition); + visit(switchCondition); + } + + List whenConditions = caseOp.getConditions(); + List thenExpressions = caseOp.getExpressions(); + + for (int i = 0; i < whenConditions.size(); i++) { + print(whenConditions.get(i)); + visit(whenConditions.get(i)); + print(thenExpressions.get(i)); + visit(thenExpressions.get(i)); + } + + MySQLExpression elseExpr = caseOp.getElseExpr(); + if (elseExpr != null) { + print(elseExpr); + visit(elseExpr); + } + } } diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 173993a4e..a34861cbd 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -1,18 +1,28 @@ package sqlancer.mysql; -import sqlancer.GlobalState; +import java.sql.SQLException; -public class MySQLGlobalState extends GlobalState { +import sqlancer.SQLGlobalState; - private MySQLSchema schema; +public class MySQLGlobalState extends SQLGlobalState { - public void setSchema(MySQLSchema schema) { - this.schema = schema; + @Override + protected MySQLSchema readSchema() throws SQLException { + return MySQLSchema.fromConnection(getConnection(), getDatabaseName()); } - public MySQLSchema getSchema() { - return schema; + public boolean usesPQS() { + return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); + } + + public boolean usesEET() { + return getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.EET || o == MySQLOracleFactory.EET_DML); + } + + public boolean usesEETDML() { + return getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.EET_DML); } } diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index c31b165a8..9219073d5 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -1,8 +1,25 @@ package sqlancer.mysql; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -@Parameters -public class MySQLOptions { +import sqlancer.DBMSSpecificOptions; + +@Parameters(separators = "=", commandDescription = "MySQL (default port: " + MySQLOptions.DEFAULT_PORT + + ", default host: " + MySQLOptions.DEFAULT_HOST + ")") +public class MySQLOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3306; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(MySQLOracleFactory.TLP_WHERE); + + @Override + public List getTestOracleFactory() { + return oracles; + } } diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java new file mode 100644 index 000000000..8b37efb13 --- /dev/null +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -0,0 +1,110 @@ +package sqlancer.mysql; + +import java.sql.SQLException; +import java.util.Optional; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.EETDMLOracle; +import sqlancer.common.oracle.EETOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.oracle.MySQLDQEOracle; +import sqlancer.mysql.oracle.MySQLDQPOracle; +import sqlancer.mysql.oracle.MySQLFuzzer; +import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; + +public enum MySQLOracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + PQS { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + + }, + CERT { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + int rowCount = rs.getInt(10); + return Optional.of((long) rowCount); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String operation = rs.getString(2); + return Optional.of(operation); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); + + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + FUZZER { + @Override + public TestOracle create(MySQLGlobalState globalState) throws Exception { + return new MySQLFuzzer(globalState); + } + + }, + DQP { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLDQPOracle(globalState); + } + }, + DQE { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLDQEOracle(globalState); + } + }, + EET { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + return new EETOracle<>(globalState, gen, expectedErrors); + } + }, + EET_DML { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()) + // The DML statements and the row-identity setup (adding/stamping the auxiliary column) touch rows, + // so they can raise the full range of DML errors — e.g. functional-index maintenance truncation — + // beyond the SELECT-based expression errors. + .with(MySQLErrors.getDMLErrors()).build(); + return new EETDMLOracle<>(globalState, gen, expectedErrors); + } + }; +} diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index ea097bf67..80a23b947 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -1,29 +1,26 @@ package sqlancer.mysql; -import java.io.FileWriter; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.List; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; -import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.StateToReproduce; -import sqlancer.StateToReproduce.MySQLStateToReproduce; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; @@ -33,6 +30,7 @@ import sqlancer.mysql.gen.MySQLSetGenerator; import sqlancer.mysql.gen.MySQLTableGenerator; import sqlancer.mysql.gen.MySQLTruncateTableGenerator; +import sqlancer.mysql.gen.MySQLUpdateGenerator; import sqlancer.mysql.gen.admin.MySQLFlush; import sqlancer.mysql.gen.admin.MySQLReset; import sqlancer.mysql.gen.datadef.MySQLIndexGenerator; @@ -41,20 +39,16 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLChecksum; import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; -import sqlancer.mysql.oracle.MySQLTLPWhereOracle; -import sqlancer.sqlite3.gen.SQLite3Common; - -public class MySQLProvider extends ProviderAdapter { - private QueryManager manager; - private String databaseName; +@AutoService(DatabaseProvider.class) +public class MySQLProvider extends SQLProviderAdapter { public MySQLProvider() { super(MySQLGlobalState.class, MySQLOptions.class); } enum Action implements AbstractAction { - SHOW_TABLES((g) -> new QueryAdapter("SHOW TABLES")), // + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), // INSERT(MySQLInsertGenerator::insertRow), // SET_VARIABLE(MySQLSetGenerator::set), // REPAIR(MySQLRepair::repair), // @@ -65,26 +59,22 @@ enum Action implements AbstractAction { FLUSH(MySQLFlush::create), RESET(MySQLReset::create), CREATE_INDEX(MySQLIndexGenerator::create), // ALTER_TABLE(MySQLAlterTable::create), // TRUNCATE_TABLE(MySQLTruncateTableGenerator::generate), // - SELECT_INFO((g) -> new QueryAdapter( + SELECT_INFO((g) -> new SQLQueryAdapter( "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() + "'")), // - CREATE_TABLE((g) -> { - // TODO refactor - String tableName = SQLite3Common.createTableName(g.getSchema().getDatabaseTables().size()); - return MySQLTableGenerator.generate(tableName, g.getRandomly(), g.getSchema()); - }), // + UPDATE(MySQLUpdateGenerator::create), // DELETE(MySQLDeleteGenerator::delete), // DROP_INDEX(MySQLDropIndex::generate); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(MySQLGlobalState globalState) throws SQLException { - return queryProvider.getQuery(globalState); + public SQLQueryAdapter getQuery(MySQLGlobalState globalState) throws Exception { + return sqlQueryProvider.getQuery(globalState); } } @@ -98,9 +88,6 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { case SHOW_TABLES: nrPerformed = r.getInteger(0, 1); break; - case CREATE_TABLE: - nrPerformed = r.getInteger(0, 1); - break; case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; @@ -138,6 +125,9 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { case SELECT_INFO: nrPerformed = r.getInteger(0, 10); break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; case DELETE: nrPerformed = r.getInteger(0, 10); break; @@ -148,106 +138,58 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { } @Override - public void generateAndTestDatabase(MySQLGlobalState globalState) throws SQLException { - this.databaseName = globalState.getDatabaseName(); - this.manager = globalState.getManager(); - Connection con = globalState.getConnection(); - MainOptions options = globalState.getOptions(); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); - Randomly r = globalState.getRandomly(); - globalState.setSchema(MySQLSchema.fromConnection(con, databaseName)); - if (options.logEachSelect()) { - logger.writeCurrent(state); - } - - while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { - String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = MySQLTableGenerator.generate(tableName, r, globalState.getSchema()); - if (options.logEachSelect()) { - logger.writeCurrent(createTable.getQueryString()); - } - manager.execute(createTable); - globalState.setSchema(MySQLSchema.fromConnection(con, databaseName)); + public void generateDatabase(MySQLGlobalState globalState) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.getNotCachedInteger(1, 2)) { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = MySQLTableGenerator.generate(globalState, tableName); + globalState.executeStatement(createTable); } StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), MySQLProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - globalState.setSchema(MySQLSchema.fromConnection(con, databaseName)); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } }); se.executeStatements(); - manager.incrementCreateDatabase(); - - // for (MySQLTable t : globalState.getSchema().getDatabaseTables()) { - // if (!ensureTableHasRows(con, t, r)) { - // return; - // } - // } - - globalState.setSchema(MySQLSchema.fromConnection(con, databaseName)); - - TestOracle oracle = new MySQLTLPWhereOracle(globalState); - for (int i = 0; i < options.getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - - } - } - - // MySQLQueryGenerator queryGenerator = new MySQLQueryGenerator(manager, r, con, databaseName); - // for (int i = 0; i < options.getNrQueries(); i++) { - // try { - // queryGenerator.generateAndCheckQuery((MySQLStateToReproduce) state, logger, options); - // } catch (IgnoreMeException e) { - // - // } - // manager.incrementSelectQueryCount(); - // } - - } - // private boolean ensureTableHasRows(Connection con, MySQLTable randomTable, Randomly r) throws SQLException { - // int nrRows; - // int counter = 1; - // do { - // try { - // Query q = MySQLRowInserter.insertRow(randomTable, r); - // manager.execute(q); - // } catch (SQLException e) { - // if (!SQLite3PivotedQuerySynthesizer.shouldIgnoreException(e)) { - // throw new AssertionError(e); - // } - // } - // nrRows = getNrRows(con, randomTable); - // } while (nrRows == 0 && counter-- != 0); - // return nrRows != 0; - // } - - public static int getNrRows(Connection con, MySQLTable table) throws SQLException { - try (Statement s = con.createStatement()) { - try (ResultSet query = s.executeQuery("SELECT COUNT(*) FROM " + table.getName())) { - query.next(); - return query.getInt(1); + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == MySQLOracleFactory.CERT)) { + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + MySQLErrors.addExpressionErrors(errors); + for (MySQLTable table : globalState.getSchema().getDatabaseTables()) { + StringBuilder sb = new StringBuilder(); + sb.append("ANALYZE TABLE "); + sb.append(table.getName()); + sb.append(" UPDATE HISTOGRAM ON "); + String columns = table.getColumns().stream().map(MySQLColumn::getName) + .collect(Collectors.joining(", ")); + sb.append(columns + ";"); + globalState.executeStatement(new SQLQueryAdapter(sb.toString(), errors)); } } } @Override - public Connection createDatabase(MySQLGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(MySQLGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = MySQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = MySQLOptions.DEFAULT_PORT; + } String databaseName = globalState.getDatabaseName(); - globalState.getState().statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName)); - globalState.getState().statements.add(new QueryAdapter("CREATE DATABASE " + databaseName)); - globalState.getState().statements.add(new QueryAdapter("USE " + databaseName)); - String url = "jdbc:mysql://localhost:3306/?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true"; - Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } @@ -257,7 +199,7 @@ public Connection createDatabase(MySQLGlobalState globalState) throws SQLExcepti try (Statement s = con.createStatement()) { s.execute("USE " + databaseName); } - return con; + return new SQLConnection(con); } @Override @@ -266,45 +208,14 @@ public String getDBMSName() { } @Override - public String toString() { - return String.format("MySQLProvider [database: %s]", databaseName); - } - - @Override - public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { - StringBuilder sb = new StringBuilder(); - MySQLStateToReproduce specificState = (MySQLStateToReproduce) state; - if (specificState.getRandomRowValues() != null) { - List columnList = specificState.getRandomRowValues().keySet().stream() - .collect(Collectors.toList()); - List tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() - .collect(Collectors.toList()); - for (MySQLTable t : tableList) { - sb.append("-- " + t.getName() + "\n"); - List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) - .collect(Collectors.toList()); - for (MySQLColumn c : columnsForTable) { - sb.append("--\t"); - sb.append(c); - sb.append("="); - sb.append(specificState.getRandomRowValues().get(c)); - sb.append("\n"); - } - } - sb.append("expected values: \n"); - sb.append(MySQLVisitor.asExpectedValues(((MySQLStateToReproduce) state).getWhereClause())); + public boolean addRowsToAllTables(MySQLGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (MySQLTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = MySQLInsertGenerator.insertRow(globalState, table); + globalState.executeStatement(queryAddRows); } - try { - writer.write(sb.toString()); - writer.flush(); - } catch (IOException e) { - throw new AssertionError(); - } - } - - @Override - public StateToReproduce getStateToReproduce(String databaseName) { - return new MySQLStateToReproduce(databaseName); + return true; } } diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 49b263248..ab0ba4543 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -1,6 +1,5 @@ package sqlancer.mysql; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; @@ -12,25 +11,30 @@ import java.util.stream.Stream; import sqlancer.Randomly; -import sqlancer.StateToReproduce.MySQLStateToReproduce; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; import sqlancer.mysql.ast.MySQLConstant; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; -public class MySQLSchema extends AbstractSchema { +public class MySQLSchema extends AbstractSchema { private static final int NR_SCHEMA_READ_TRIES = 10; public enum MySQLDataType { INT, VARCHAR, FLOAT, DOUBLE, DECIMAL; - public static MySQLDataType getRandom() { - return Randomly.fromOptions(values()); + public static MySQLDataType getRandom(MySQLGlobalState globalState) { + if (globalState.usesPQS()) { + return Randomly.fromOptions(MySQLDataType.INT, MySQLDataType.VARCHAR); + } else { + return Randomly.fromOptions(values()); + } } public boolean isNumeric() { @@ -53,6 +57,7 @@ public static class MySQLColumn extends AbstractTableColumn tables) { super(tables); } - public MySQLRowValue getRandomRowValue(Connection con, MySQLStateToReproduce state) throws SQLException { + public MySQLRowValue getRandomRowValue(SQLConnection con) throws SQLException { String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + @@ -94,22 +105,14 @@ public MySQLRowValue getRandomRowValue(Connection con, MySQLStateToReproduce sta try (Statement s = con.createStatement()) { ResultSet randomRowValues = s.executeQuery(randomRow); if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow + "\n" + state); + throw new AssertionError("could not find random row! " + randomRow + "\n"); } for (int i = 0; i < getColumns().size(); i++) { MySQLColumn column = getColumns().get(i); Object value; int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); assert columnIndex == i + 1; - // String typeString = randomRowValues.getString(columnIndex + getColumns().size()); - // MySQLDataType valueType = getColumnType(typeString); MySQLConstant constant; - // if (randomRowValues.getString(columnIndex) == null) { - // value = null; - // constant = MySQLConstant.createNullConstant(); - // } else { - // switch (valueType) { - // case INT: if (randomRowValues.getString(columnIndex) == null) { constant = MySQLConstant.createNullConstant(); } else { @@ -126,15 +129,9 @@ public MySQLRowValue getRandomRowValue(Connection con, MySQLStateToReproduce sta throw new AssertionError(column.getType()); } } - // break; - // default: - // throw new AssertionError(valueType); - // } - // } values.put(column, constant); } assert !randomRowValues.next(); - state.randomRowValues = values; return new MySQLRowValue(this, values); } @@ -167,60 +164,15 @@ private static MySQLDataType getColumnType(String typeString) { } } - public static class MySQLRowValue { - - private final MySQLTables tables; - private final Map values; + public static class MySQLRowValue extends AbstractRowValue { MySQLRowValue(MySQLTables tables, Map values) { - this.tables = tables; - this.values = values; - } - - public MySQLTables getTable() { - return tables; - } - - public Map getValues() { - return values; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - int i = 0; - for (MySQLColumn c : tables.getColumns()) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(values.get(c)); - } - return sb.toString(); - } - - public String getRowValuesAsString() { - List columnsToCheck = tables.getColumns(); - return getRowValuesAsString(columnsToCheck); - } - - public String getRowValuesAsString(List columnsToCheck) { - StringBuilder sb = new StringBuilder(); - Map expectedValues = getValues(); - for (int i = 0; i < columnsToCheck.size(); i++) { - if (i != 0) { - sb.append(", "); - } - MySQLConstant expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); - visitor.visit(expectedColumnValue); - sb.append(visitor.get()); - } - return sb.toString(); + super(tables, values); } } - public static class MySQLTable extends AbstractTable { + public static class MySQLTable extends AbstractRelationalTable { public enum MySQLEngine { INNO_DB("InnoDB"), MY_ISAM("MyISAM"), MEMORY("MEMORY"), HEAP("HEAP"), CSV("CSV"), MERGE("MERGE"), @@ -232,10 +184,6 @@ public enum MySQLEngine { this.s = s; } - String getTextRepresentation() { - return s; - } - public static MySQLEngine get(String val) { return Stream.of(values()).filter(engine -> engine.s.equalsIgnoreCase(val)).findFirst().get(); } @@ -253,10 +201,6 @@ public MySQLEngine getEngine() { return engine; } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } public static final class MySQLIndex extends TableIndex { @@ -280,7 +224,7 @@ public String getIndexName() { } - public static MySQLSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static MySQLSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { Exception ex = null; /* the loop is a workaround for https://bugs.mysql.com/bug.php?id=95929 */ for (int i = 0; i < NR_SCHEMA_READ_TRIES; i++) { @@ -312,7 +256,7 @@ public static MySQLSchema fromConnection(Connection con, String databaseName) th throw new AssertionError(ex); } - private static List getIndexes(Connection con, String tableName, String databaseName) + private static List getIndexes(SQLConnection con, String tableName, String databaseName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -328,7 +272,7 @@ private static List getIndexes(Connection con, String tableName, Str return indexes; } - private static List getTableColumns(Connection con, String tableName, String databaseName) + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { @@ -338,8 +282,10 @@ private static List getTableColumns(Connection con, String tableNam String columnName = rs.getString("COLUMN_NAME"); String dataType = rs.getString("DATA_TYPE"); int precision = rs.getInt("NUMERIC_PRECISION"); + int scale = rs.getInt("NUMERIC_SCALE"); boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); - MySQLColumn c = new MySQLColumn(columnName, getColumnType(dataType), isPrimaryKey, precision); + MySQLColumn c = new MySQLColumn(columnName, getColumnType(dataType), isPrimaryKey, precision, + scale); columns.add(c); } } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index af34daaa4..be82d45b5 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -4,10 +4,14 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -16,13 +20,14 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.visitor.ToStringVisitor; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { @@ -36,6 +41,11 @@ public void visitSpecific(MySQLExpression expr) { @Override public void visit(MySQLSelect s) { sb.append("SELECT "); + if (s.getHint() != null) { + sb.append("/*+ "); + visit(s.getHint()); + sb.append("*/ "); + } switch (s.getFromOptions()) { case DISTINCT: sb.append("DISTINCT "); @@ -50,7 +60,7 @@ public void visit(MySQLSelect s) { throw new AssertionError(); } sb.append(s.getModifiers().stream().collect(Collectors.joining(" "))); - if (s.getModifiers().size() > 0) { + if (!s.getModifiers().isEmpty()) { sb.append(" "); } if (s.getFetchColumns() == null) { @@ -83,7 +93,7 @@ public void visit(MySQLSelect s) { sb.append(" WHERE "); visit(whereClause); } - if (s.getGroupByExpressions() != null && s.getGroupByExpressions().size() > 0) { + if (s.getGroupByExpressions() != null && !s.getGroupByExpressions().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); List groupBys = s.getGroupByExpressions(); @@ -94,14 +104,14 @@ public void visit(MySQLSelect s) { visit(groupBys.get(i)); } } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - List orderBys = s.getOrderByExpressions(); + List orderBys = s.getOrderByClauses(); for (int i = 0; i < orderBys.size(); i++) { if (i != 0) { sb.append(", "); } - visit(s.getOrderByExpressions().get(i)); + visit(s.getOrderByClauses().get(i)); } } if (s.getLimitClause() != null) { @@ -278,4 +288,88 @@ public void visit(MySQLCollate collate) { sb.append(")"); } + @Override + public void visit(MySQLJoin join) { + sb.append(" "); + switch (join.getType()) { + case NATURAL: + sb.append("NATURAL "); + break; + case INNER: + sb.append("INNER "); + break; + case STRAIGHT: + sb.append("STRAIGHT_"); + break; + case LEFT: + sb.append("LEFT "); + break; + case RIGHT: + sb.append("RIGHT "); + break; + case CROSS: + sb.append("CROSS "); + break; + default: + throw new AssertionError(join.getType()); + } + sb.append("JOIN "); + sb.append(join.getTable().getName()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + @Override + public void visit(MySQLText text) { + sb.append(text.getText()); + } + + @Override + public void visit(MySQLAggregate aggr) { + MySQLAggregateFunction func = aggr.getFunc(); + String option = func.getOption(); + List exprs = aggr.getExprs(); + + sb.append(func.getName()); + sb.append("("); + if (option != null) { + sb.append(option); + sb.append(" "); + } + for (int i = 0; i < exprs.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(exprs.get(i)); + } + sb.append(")"); + } + + @Override + public void visit(MySQLCaseOperator caseOp) { + sb.append("(CASE "); + + if (caseOp.getSwitchCondition() != null) { + visit(caseOp.getSwitchCondition()); + sb.append(" "); + } + + for (int i = 0; i < caseOp.getConditions().size(); i++) { + if (i > 0) { + sb.append(" "); + } + sb.append("WHEN "); + visit(caseOp.getConditions().get(i)); + sb.append(" THEN "); + visit(caseOp.getExpressions().get(i)); + } + + if (caseOp.getElseExpr() != null) { + sb.append(" ELSE "); + visit(caseOp.getElseExpr()); + } + sb.append(" END)"); + } } diff --git a/src/sqlancer/mysql/MySQLVisitor.java b/src/sqlancer/mysql/MySQLVisitor.java index eeda4f681..12c93ecfc 100644 --- a/src/sqlancer/mysql/MySQLVisitor.java +++ b/src/sqlancer/mysql/MySQLVisitor.java @@ -1,9 +1,11 @@ package sqlancer.mysql; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -12,10 +14,12 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; public interface MySQLVisitor { @@ -52,6 +56,14 @@ public interface MySQLVisitor { void visit(MySQLCollate collate); + void visit(MySQLJoin join); + + void visit(MySQLText text); + + void visit(MySQLAggregate aggregate); + + void visit(MySQLCaseOperator caseOp); + default void visit(MySQLExpression expr) { if (expr instanceof MySQLConstant) { visit((MySQLConstant) expr); @@ -77,6 +89,8 @@ default void visit(MySQLExpression expr) { visit((MySQLOrderByTerm) expr); } else if (expr instanceof MySQLExists) { visit((MySQLExists) expr); + } else if (expr instanceof MySQLJoin) { + visit((MySQLJoin) expr); } else if (expr instanceof MySQLStringExpression) { visit((MySQLStringExpression) expr); } else if (expr instanceof MySQLBetweenOperation) { @@ -85,6 +99,12 @@ default void visit(MySQLExpression expr) { visit((MySQLTableReference) expr); } else if (expr instanceof MySQLCollate) { visit((MySQLCollate) expr); + } else if (expr instanceof MySQLText) { + visit((MySQLText) expr); + } else if (expr instanceof MySQLAggregate) { + visit((MySQLAggregate) expr); + } else if (expr instanceof MySQLCaseOperator) { + visit((MySQLCaseOperator) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java new file mode 100644 index 000000000..94c4e426b --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -0,0 +1,55 @@ +package sqlancer.mysql.ast; + +import java.util.List; + +public class MySQLAggregate implements MySQLExpression { + + public enum MySQLAggregateFunction { + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_count. + COUNT("COUNT", null, false), COUNT_DISTINCT("COUNT", "DISTINCT", true), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_sum. + SUM("SUM", null, false), SUM_DISTINCT("SUM", "DISTINCT", false), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_min. + MIN("MIN", null, false), MIN_DISTINCT("MIN", "DISTINCT", false), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. + MAX("MAX", null, false), MAX_DISTINCT("MAX", "DISTINCT", false); + + private final String name; + private final String option; + private final boolean isVariadic; + + MySQLAggregateFunction(String name, String option, boolean isVariadic) { + this.name = name; + this.option = option; + this.isVariadic = isVariadic; + } + + public String getName() { + return this.name; + } + + public String getOption() { + return option; + } + + public boolean isVariadic() { + return this.isVariadic; + } + } + + private final List exprs; + private final MySQLAggregateFunction func; + + public MySQLAggregate(List exprs, MySQLAggregateFunction func) { + this.exprs = exprs; + this.func = func; + } + + public List getExprs() { + return exprs; + } + + public MySQLAggregateFunction getFunc() { + return func; + } +} diff --git a/src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java b/src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java index 4971ccc49..102e01b0d 100644 --- a/src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java +++ b/src/sqlancer/mysql/ast/MySQLBinaryLogicalOperation.java @@ -101,7 +101,9 @@ public String getTextRepresentation() { public MySQLConstant getExpectedValue() { MySQLConstant leftExpected = left.getExpectedValue(); MySQLConstant rightExpected = right.getExpectedValue(); - + if (left.getExpectedValue() == null || right.getExpectedValue() == null) { + return null; + } return op.apply(leftExpected, rightExpected); } diff --git a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java index 33293f26b..af0b0ccf7 100644 --- a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java +++ b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java @@ -78,20 +78,20 @@ public MySQLConstant getExpectedValue() { /* workaround for https://bugs.mysql.com/bug.php?id=95960 */ if (leftExpected.isString()) { String text = leftExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && text.length() > 0) { + while (text.startsWith(" ") || text.startsWith("\t")) { text = text.substring(1); } - if (text.length() > 0 && (text.startsWith("\n") || text.startsWith("."))) { + if (text.startsWith("\n") || text.startsWith(".")) { throw new IgnoreMeException(); } } if (rightExpected.isString()) { String text = rightExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && text.length() > 0) { + while (text.startsWith(" ") || text.startsWith("\t")) { text = text.substring(1); } - if (text.length() > 0 && (text.startsWith("\n") || text.startsWith("."))) { + if (text.startsWith("\n") || text.startsWith(".")) { throw new IgnoreMeException(); } } diff --git a/src/sqlancer/mysql/ast/MySQLCaseOperator.java b/src/sqlancer/mysql/ast/MySQLCaseOperator.java new file mode 100644 index 000000000..ceb7fbbaa --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLCaseOperator.java @@ -0,0 +1,48 @@ +package sqlancer.mysql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class MySQLCaseOperator extends NewCaseOperatorNode implements MySQLExpression { + + public MySQLCaseOperator(MySQLExpression switchCondition, List whenExprs, + List thenExprs, MySQLExpression elseExpr) { + super(switchCondition, whenExprs, thenExprs, elseExpr); + } + + @Override + public MySQLConstant getExpectedValue() { + int nrConditions = getConditions().size(); + + MySQLExpression switchCondition = getSwitchCondition(); + List whenExprs = getConditions(); + List thenExprs = getExpressions(); + MySQLExpression elseExpr = getElseExpr(); + + if (switchCondition != null) { + MySQLConstant switchValue = switchCondition.getExpectedValue(); + + for (int i = 0; i < nrConditions; i++) { + MySQLConstant whenValue = whenExprs.get(i).getExpectedValue(); + MySQLConstant isConditionMatched = switchValue.isEquals(whenValue); + if (!isConditionMatched.isNull() && isConditionMatched.asBooleanNotNull()) { + return thenExprs.get(i).getExpectedValue(); + } + } + } else { + for (int i = 0; i < nrConditions; i++) { + MySQLConstant whenValue = whenExprs.get(i).getExpectedValue(); + if (!whenValue.isNull() && whenValue.asBooleanNotNull()) { + return thenExprs.get(i).getExpectedValue(); + } + } + } + + if (elseExpr != null) { + return elseExpr.getExpectedValue(); + } + + return MySQLConstant.createNullConstant(); + } +} diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index 8ae783fa0..07c45a429 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -1,16 +1,89 @@ package sqlancer.mysql.ast; +import java.util.Objects; + public class MySQLCastOperation implements MySQLExpression { private final MySQLExpression expr; private final CastType type; - public enum CastType { - SIGNED, UNSIGNED; + /** + * A MySQL {@code CAST} target type. The non-{@code DECIMAL} kinds are interned singletons; {@code DECIMAL} may + * additionally carry an {@code (M, D)} precision/scale (via {@link #decimal}) so that a + * {@code CAST(... AS DECIMAL(M, D))} can reproduce a column's exact type. This is relied on by the EET oracle's + * type-pinning casts (see {@code MySQLEETTransformer}). + */ + public static final class CastType { + + // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never evaluated, + // so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + public static final CastType SIGNED = new CastType(Kind.SIGNED); + public static final CastType UNSIGNED = new CastType(Kind.UNSIGNED); + public static final CastType CHAR = new CastType(Kind.CHAR); + public static final CastType FLOAT = new CastType(Kind.FLOAT); + public static final CastType DOUBLE = new CastType(Kind.DOUBLE); + public static final CastType DECIMAL = new CastType(Kind.DECIMAL); + + private enum Kind { + SIGNED, UNSIGNED, CHAR, FLOAT, DOUBLE, DECIMAL + } + + private final Kind kind; + private final Integer precision; // DECIMAL only, otherwise null + private final Integer scale; // DECIMAL only, otherwise null + + private CastType(Kind kind) { + this(kind, null, null); + } + + private CastType(Kind kind, Integer precision, Integer scale) { + this.kind = kind; + this.precision = precision; + this.scale = scale; + } + + // A DECIMAL(precision, scale) cast target. + public static CastType decimal(int precision, int scale) { + return new CastType(Kind.DECIMAL, precision, scale); + } public static CastType getRandom() { return SIGNED; - // return Randomly.fromOptions(CastType.values()); + // return Randomly.fromOptions(CastType.SIGNED, CastType.UNSIGNED); + } + + public Integer getPrecision() { + return precision; + } + + public Integer getScale() { + return scale; + } + + @Override + public String toString() { + if (precision == null) { + return kind.name(); + } + return kind.name() + "(" + precision + ", " + scale + ")"; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof CastType)) { + return false; + } + CastType other = (CastType) obj; + return kind == other.kind && Objects.equals(precision, other.precision) + && Objects.equals(scale, other.scale); + } + + @Override + public int hashCode() { + return Objects.hash(kind, precision, scale); } } diff --git a/src/sqlancer/mysql/ast/MySQLCollate.java b/src/sqlancer/mysql/ast/MySQLCollate.java index a0dc5c4e0..4512486f4 100644 --- a/src/sqlancer/mysql/ast/MySQLCollate.java +++ b/src/sqlancer/mysql/ast/MySQLCollate.java @@ -1,6 +1,6 @@ package sqlancer.mysql.ast; -import sqlancer.ast.UnaryNode; +import sqlancer.common.ast.UnaryNode; public class MySQLCollate extends UnaryNode implements MySQLExpression { diff --git a/src/sqlancer/mysql/ast/MySQLComputableFunction.java b/src/sqlancer/mysql/ast/MySQLComputableFunction.java index d049a8ba3..7ffc93000 100644 --- a/src/sqlancer/mysql/ast/MySQLComputableFunction.java +++ b/src/sqlancer/mysql/ast/MySQLComputableFunction.java @@ -38,12 +38,13 @@ public enum MySQLFunction { // } // }, /** - * @see https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#function_bit-count + * @see Bit Functions + * and Operators */ BIT_COUNT(1, "BIT_COUNT") { @Override - public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression[] args) { + public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression... args) { MySQLConstant arg = evaluatedArgs[0]; if (arg.isNull()) { return MySQLConstant.createNullConstant(); @@ -74,7 +75,7 @@ public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression[] args COALESCE(2, "COALESCE") { @Override - public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { + public MySQLConstant apply(MySQLConstant[] args, MySQLExpression... origArgs) { MySQLConstant result = MySQLConstant.createNullConstant(); for (MySQLConstant arg : args) { if (!arg.isNull()) { @@ -92,12 +93,13 @@ public boolean isVariadic() { }, /** - * @see https://dev.mysql.com/doc/refman/8.0/en/control-flow-functions.html#function_if + * @see Flow Control + * Functions */ IF(3, "IF") { @Override - public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { + public MySQLConstant apply(MySQLConstant[] args, MySQLExpression... origArgs) { MySQLConstant cond = args[0]; MySQLConstant left = args[1]; MySQLConstant right = args[2]; @@ -113,12 +115,12 @@ public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { }, /** - * @see https://dev.mysql.com/doc/refman/8.0/en/control-flow-functions.html#function_ifnull + * @see IFNULL */ IFNULL(2, "IFNULL") { @Override - public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { + public MySQLConstant apply(MySQLConstant[] args, MySQLExpression... origArgs) { MySQLConstant result; if (args[0].isNull()) { result = args[1]; @@ -132,14 +134,14 @@ public MySQLConstant apply(MySQLConstant[] args, MySQLExpression[] origArgs) { LEAST(2, "LEAST", true) { @Override - public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression[] args) { + public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression... args) { return aggregate(evaluatedArgs, (min, cur) -> cur.isLessThan(min).asBooleanNotNull() ? cur : min); } }, GREATEST(2, "GREATEST", true) { @Override - public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression[] args) { + public MySQLConstant apply(MySQLConstant[] evaluatedArgs, MySQLExpression... args) { return aggregate(evaluatedArgs, (max, cur) -> cur.isLessThan(max).asBooleanNotNull() ? max : cur); } }; @@ -177,6 +179,8 @@ private static MySQLConstant aggregate(MySQLConstant[] evaluatedArgs, BinaryOper /** * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number * of arguments is returned. + * + * @return the number of arguments */ public int getNrArgs() { return nrArgs; @@ -207,6 +211,9 @@ public MySQLConstant getExpectedValue() { MySQLConstant[] constants = new MySQLConstant[args.length]; for (int i = 0; i < constants.length; i++) { constants[i] = args[i].getExpectedValue(); + if (constants[i].getExpectedValue() == null) { + return null; + } } return func.apply(constants, args); } diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index e04b54c40..5fb0698b9 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -30,8 +30,7 @@ private RuntimeException throwException() { @Override public MySQLConstant isEquals(MySQLConstant rightVal) { - throw throwException(); - + return null; } @Override @@ -69,6 +68,11 @@ public MySQLDoubleConstant(double val) { } } + @Override + public double getDouble() { + return val; + } + @Override public String getTextRepresentation() { return String.valueOf(val); @@ -165,12 +169,12 @@ public MySQLConstant castAs(CastType type) { try { String substring = value.substring(0, i); long val = Long.parseLong(substring); - return MySQLConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); + return MySQLConstant.createIntConstant(val, type == CastType.SIGNED); } catch (NumberFormatException e) { // ignore } } - return MySQLConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); + return MySQLConstant.createIntConstant(0, type == CastType.SIGNED); } else { throw new AssertionError(); } @@ -199,7 +203,9 @@ protected MySQLConstant isLessThan(MySQLConstant rightVal) { return castAs(rightVal.isSigned() ? CastType.SIGNED : CastType.UNSIGNED).isLessThan(rightVal); } else if (rightVal.isString()) { // unexpected result for '-' < "!"; - // return MySQLConstant.createBoolean(value.compareToIgnoreCase(rightVal.getString()) < 0); + // return + // MySQLConstant.createBoolean(value.compareToIgnoreCase(rightVal.getString()) < + // 0); throw new IgnoreMeException(); } else { throw new AssertionError(rightVal); @@ -217,16 +223,10 @@ public static class MySQLIntConstant extends MySQLConstant { public MySQLIntConstant(long value, boolean isSigned) { this.value = value; this.isSigned = isSigned; - if (value == 0 && Randomly.getBoolean()) { - stringRepresentation = "FALSE"; - } else if (value == 1 && Randomly.getBoolean()) { - stringRepresentation = "TRUE"; + if (isSigned) { + stringRepresentation = String.valueOf(value); } else { - if (isSigned) { - stringRepresentation = String.valueOf(value); - } else { - stringRepresentation = Long.toUnsignedString(value); - } + stringRepresentation = Long.toUnsignedString(value); } } @@ -386,6 +386,10 @@ public long getInt() { throw new UnsupportedOperationException(); } + public double getDouble() { + throw new UnsupportedOperationException(); + } + public boolean isSigned() { return false; } @@ -446,16 +450,6 @@ public String toString() { public abstract MySQLConstant isEquals(MySQLConstant rightVal); - public MySQLConstant isEqualsNullSafe(MySQLConstant rightVal) { - if (isNull()) { - return MySQLConstant.createBoolean(rightVal.isNull()); - } else if (rightVal.isNull()) { - return MySQLConstant.createFalse(); - } else { - return isEquals(rightVal); - } - } - public abstract MySQLConstant castAs(CastType type); public abstract String castAsString(); diff --git a/src/sqlancer/mysql/ast/MySQLExpression.java b/src/sqlancer/mysql/ast/MySQLExpression.java index 61a3b8aeb..1f3ae5bcb 100644 --- a/src/sqlancer/mysql/ast/MySQLExpression.java +++ b/src/sqlancer/mysql/ast/MySQLExpression.java @@ -1,6 +1,9 @@ package sqlancer.mysql.ast; -public interface MySQLExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.mysql.MySQLSchema.MySQLColumn; + +public interface MySQLExpression extends Expression { default MySQLConstant getExpectedValue() { throw new AssertionError("PQS not supported for this operator"); diff --git a/src/sqlancer/mysql/ast/MySQLInOperation.java b/src/sqlancer/mysql/ast/MySQLInOperation.java index 93d340063..81dff62f2 100644 --- a/src/sqlancer/mysql/ast/MySQLInOperation.java +++ b/src/sqlancer/mysql/ast/MySQLInOperation.java @@ -5,7 +5,8 @@ import sqlancer.IgnoreMeException; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_in + * @see Comparison Functions and + * Operators */ public class MySQLInOperation implements MySQLExpression { diff --git a/src/sqlancer/mysql/ast/MySQLJoin.java b/src/sqlancer/mysql/ast/MySQLJoin.java index 8558e43a9..c063b4f1b 100644 --- a/src/sqlancer/mysql/ast/MySQLJoin.java +++ b/src/sqlancer/mysql/ast/MySQLJoin.java @@ -1,10 +1,87 @@ package sqlancer.mysql.ast; -public class MySQLJoin implements MySQLExpression { +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + +public class MySQLJoin implements MySQLExpression, Join { + + public enum JoinType { + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; + } + + private final MySQLTable table; + private MySQLExpression onClause; + private JoinType type; + + public MySQLJoin(MySQLJoin other) { + this.table = other.table; + this.onClause = other.onClause; + this.type = other.type; + } + + public MySQLJoin(MySQLTable table, MySQLExpression onClause, JoinType type) { + this.table = table; + this.onClause = onClause; + this.type = type; + } + + public MySQLTable getTable() { + return table; + } + + public MySQLExpression getOnClause() { + return onClause; + } + + public JoinType getType() { + return type; + } @Override - public MySQLConstant getExpectedValue() { - throw new UnsupportedOperationException(); + public void setOnClause(MySQLExpression onClause) { + this.onClause = onClause; } + public void setType(JoinType type) { + this.type = type; + } + + public static List getRandomJoinClauses(List tables, MySQLGlobalState globalState) { + List joinStatements = new ArrayList<>(); + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + List columns = new ArrayList<>(); + if (tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + MySQLTable table = Randomly.fromList(tables); + tables.remove(table); + columns.addAll(table.getColumns()); + MySQLExpressionGenerator joinGen = new MySQLExpressionGenerator(globalState).setColumns(columns); + MySQLExpression joinClause = joinGen.generateExpression(); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + joinClause = null; + } + MySQLJoin j = new MySQLJoin(table, joinClause, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } } diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 8e48f7dac..7b9243c20 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -2,13 +2,20 @@ import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; -import sqlancer.ast.SelectBase; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; -public class MySQLSelect extends SelectBase implements MySQLExpression { +public class MySQLSelect extends SelectBase + implements MySQLExpression, Select { private SelectType fromOptions = SelectType.ALL; private List modifiers = Collections.emptyList(); + private MySQLText hint; public enum SelectType { DISTINCT, ALL, DISTINCTROW; @@ -39,4 +46,28 @@ public MySQLConstant getExpectedValue() { return null; } + public void setHint(MySQLText hint) { + this.hint = hint; + } + + public MySQLText getHint() { + return hint; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (MySQLExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (MySQLJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return MySQLVisitor.asString(this); + } } diff --git a/src/sqlancer/mysql/ast/MySQLText.java b/src/sqlancer/mysql/ast/MySQLText.java new file mode 100644 index 000000000..36040a383 --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLText.java @@ -0,0 +1,14 @@ +package sqlancer.mysql.ast; + +public class MySQLText implements MySQLExpression { + + private final String text; + + public MySQLText(String text) { + this.text = text; + } + + public String getText() { + return text; + } +} diff --git a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java index 6bf362f2e..45a1fe888 100644 --- a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java +++ b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java @@ -2,8 +2,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLUnaryPrefixOperation extends UnaryOperatorNode @@ -62,6 +62,10 @@ public MySQLUnaryPrefixOperation(MySQLExpression expr, MySQLUnaryPrefixOperator super(expr, op); } + public MySQLUnaryPrefixOperator getOp() { + return op; + } + @Override public MySQLConstant getExpectedValue() { MySQLConstant subExprVal = expr.getExpectedValue(); diff --git a/src/sqlancer/mysql/gen/MySQLAlterTable.java b/src/sqlancer/mysql/gen/MySQLAlterTable.java index 22b22dee1..f2c952016 100644 --- a/src/sqlancer/mysql/gen/MySQLAlterTable.java +++ b/src/sqlancer/mysql/gen/MySQLAlterTable.java @@ -5,9 +5,9 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -24,7 +24,7 @@ public MySQLAlterTable(MySQLSchema newSchema) { this.schema = newSchema; } - public static Query create(MySQLGlobalState globalState) { + public static SQLQueryAdapter create(MySQLGlobalState globalState) { return new MySQLAlterTable(globalState.getSchema()).create(); } @@ -60,10 +60,9 @@ PACK_KEYS, RENAME("doesn't exist", "already exists"), /* WITH_WITHOUT_VALIDATION } - private Query create() { - List errors = new ArrayList<>( - Arrays.asList("does not support the create option", "doesn't have this option", - "is not supported for this operation", "Data truncation", "Specified key was too long")); + private SQLQueryAdapter create() { + ExpectedErrors errors = ExpectedErrors.from("does not support the create option", "doesn't have this option", + "is not supported for this operation", "Data truncation", "Specified key was too long"); errors.add("Data truncated for functional index "); sb.append("ALTER TABLE "); MySQLTable table = schema.getRandomTable(); @@ -131,15 +130,15 @@ private Query create() { break; case STATS_AUTO_RECALC: sb.append("STATS_AUTO_RECALC "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; case STATS_PERSISTENT: sb.append("STATS_PERSISTENT "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; case PACK_KEYS: sb.append("PACK_KEYS "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; // not relevant: // case WITH_WITHOUT_VALIDATION: @@ -152,7 +151,8 @@ private Query create() { sb.append(Randomly.fromOptions("TO", "AS")); sb.append(" "); } - sb.append("t" + Randomly.smallNumber()); + sb.append("t"); + sb.append(Randomly.smallNumber()); couldAffectSchema = true; break; default: @@ -173,7 +173,7 @@ private Query create() { errors.add(error); } } - return new QueryAdapter(sb.toString(), errors, couldAffectSchema); + return new SQLQueryAdapter(sb.toString(), errors, couldAffectSchema); } } diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index 83c591d25..551264c4a 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -1,34 +1,31 @@ package sqlancer.mysql.gen; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; -public class MySQLDeleteGenerator { +public class MySQLDeleteGenerator extends AbstractDeleteGenerator { - private final StringBuilder sb = new StringBuilder(); private final MySQLGlobalState globalState; public MySQLDeleteGenerator(MySQLGlobalState globalState) { this.globalState = globalState; } - public static Query delete(MySQLGlobalState globalState) { - return new MySQLDeleteGenerator(globalState).generate(); + public static SQLQueryAdapter delete(MySQLGlobalState globalState) { + return new MySQLDeleteGenerator(globalState).getStatement(); } - private Query generate() { + @Override + public void buildStatement() { MySQLTable randomTable = globalState.getSchema().getRandomTable(); MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()); - Set errors = new HashSet<>(); sb.append("DELETE"); if (Randomly.getBoolean()) { sb.append(" LOW_PRIORITY"); @@ -43,8 +40,7 @@ private Query generate() { sb.append(" FROM "); sb.append(randomTable.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(MySQLVisitor.asString(gen.generateExpression())); + appendWhereClause(MySQLVisitor.asString(gen.generateExpression())); MySQLErrors.addExpressionErrors(errors); } errors.addAll(Arrays.asList("doesn't have this option", @@ -53,7 +49,6 @@ private Query generate() { */, "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index")); // TODO: support ORDER BY - return new QueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mysql/gen/MySQLDropIndex.java b/src/sqlancer/mysql/gen/MySQLDropIndex.java index 9e3278fa5..569a5bfec 100644 --- a/src/sqlancer/mysql/gen/MySQLDropIndex.java +++ b/src/sqlancer/mysql/gen/MySQLDropIndex.java @@ -1,16 +1,14 @@ package sqlancer.mysql.gen; -import java.util.Arrays; - import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/drop-index.html + * @see DROP INDEX Statement */ public final class MySQLDropIndex { @@ -26,7 +24,7 @@ private MySQLDropIndex() { // lock_option: // LOCK [=] {DEFAULT|NONE|SHARED|EXCLUSIVE} - public static Query generate(MySQLGlobalState globalState) { + public static SQLQueryAdapter generate(MySQLGlobalState globalState) { MySQLTable table = globalState.getSchema().getRandomTable(); if (!table.hasIndexes()) { throw new IgnoreMeException(); @@ -44,9 +42,10 @@ public static Query generate(MySQLGlobalState globalState) { sb.append(" LOCK="); sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); } - return new QueryAdapter(sb.toString(), - Arrays.asList("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", "Data truncation", - "Data truncated for functional index", "A primary key index cannot be invisible")); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", + "Data truncation", "Data truncated for functional index", + "A primary key index cannot be invisible")); } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index fb8baf108..baea11f65 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -1,15 +1,30 @@ package sqlancer.mysql.gen; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.gen.UntypedExpressionGenerator; +import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.EETDMLGenerator; +import sqlancer.common.gen.EETGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.oracle.EETTransformer; +import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; @@ -17,6 +32,7 @@ import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; import sqlancer.mysql.ast.MySQLBinaryOperation; import sqlancer.mysql.ast.MySQLBinaryOperation.MySQLBinaryOperator; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLComputableFunction; @@ -26,15 +42,26 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; +import sqlancer.mysql.ast.MySQLOrderByTerm; +import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; +import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; +import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.oracle.MySQLEETTransformer; -public class MySQLExpressionGenerator extends UntypedExpressionGenerator { +public class MySQLExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator, + CERTGenerator, + EETGenerator, + EETDMLGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; + private List tables; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -47,7 +74,7 @@ public MySQLExpressionGenerator setRowVal(MySQLRowValue rowVal) { private enum Actions { COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, - BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, BINARY_OPERATION, EXISTS, BETWEEN_OPERATOR; + BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, BINARY_OPERATION, EXISTS, BETWEEN_OPERATOR, CASE_OPERATOR; } @Override @@ -63,10 +90,6 @@ public MySQLExpression generateExpression(int depth) { case UNARY_PREFIX_OPERATION: MySQLExpression subExpr = generateExpression(depth + 1); MySQLUnaryPrefixOperator random = MySQLUnaryPrefixOperator.getRandom(); - if (random == MySQLUnaryPrefixOperator.MINUS) { - // workaround for https://bugs.mysql.com/bug.php?id=99122 - throw new IgnoreMeException(); - } return new MySQLUnaryPrefixOperation(subExpr, random); case UNARY_POSTFIX: return new MySQLUnaryPostfixOperation(generateExpression(depth + 1), @@ -98,12 +121,16 @@ public MySQLExpression generateExpression(int depth) { case EXISTS: return getExists(); case BETWEEN_OPERATOR: - if (MySQLBugs.bug99181) { + if (MySQLBugs.bug99182) { // TODO: there are a number of bugs that are triggered by the BETWEEN operator throw new IgnoreMeException(); } return new MySQLBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1)); + case CASE_OPERATOR: + int nr = Randomly.smallNumber() + 1; + return new MySQLCaseOperator(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); default: throw new AssertionError(); } @@ -132,44 +159,31 @@ private MySQLExpression getComputableFunction(int depth) { private enum ConstantType { INT, NULL, STRING, DOUBLE; + + public static ConstantType[] valuesPQS() { + return new ConstantType[] { INT, NULL, STRING }; + } } @Override public MySQLExpression generateConstant() { - switch (Randomly.fromOptions(ConstantType.values())) { + ConstantType[] values; + if (state.usesPQS()) { + values = ConstantType.valuesPQS(); + } else { + values = ConstantType.values(); + } + switch (Randomly.fromOptions(values)) { case INT: return MySQLConstant.createIntConstant((int) state.getRandomly().getInteger()); case NULL: return MySQLConstant.createNullConstant(); case STRING: - String string = state.getRandomly().getString(); - if (string.startsWith("\n")) { - // workaround for https://bugs.mysql.com/bug.php?id=99130 - throw new IgnoreMeException(); - } - if (string.startsWith("-0") || string.startsWith("0.") || string.startsWith(".")) { - // https://bugs.mysql.com/bug.php?id=99145 - throw new IgnoreMeException(); - } - MySQLConstant createStringConstant = MySQLConstant.createStringConstant(string); - // if (Randomly.getBoolean()) { - // return new MySQLCollate(createStringConstant, Randomly.fromOptions("ascii_bin", "binary")); - // } - if (string.startsWith("1e")) { - // https://bugs.mysql.com/bug.php?id=99146 - throw new IgnoreMeException(); - } - return createStringConstant; + /* Replace characters that still trigger open bugs in MySQL */ + String string = state.getRandomly().getString().replace("\\", "").replace("\n", ""); + return MySQLConstant.createStringConstant(string); case DOUBLE: double val = state.getRandomly().getDouble(); - if (Math.abs(val) <= 1 && val != 0) { - // https://bugs.mysql.com/bug.php?id=99145 - throw new IgnoreMeException(); - } - if (Math.abs(val) > 1.0E30) { - // https://bugs.mysql.com/bug.php?id=99146 - throw new IgnoreMeException(); - } return new MySQLDoubleConstant(val); default: throw new AssertionError(); @@ -188,4 +202,226 @@ protected MySQLExpression generateColumn() { return MySQLColumnReference.create(c, val); } + @Override + public MySQLExpression negatePredicate(MySQLExpression predicate) { + return new MySQLUnaryPrefixOperation(predicate, MySQLUnaryPrefixOperator.NOT); + } + + @Override + public MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, MySQLUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + } + + @Override + public List generateOrderBys() { + List expressions = super.generateOrderBys(); + List newOrderBys = new ArrayList<>(); + for (MySQLExpression expr : expressions) { + if (Randomly.getBoolean()) { + MySQLOrderByTerm newExpr = new MySQLOrderByTerm(expr, MySQLOrder.getRandomOrder()); + newOrderBys.add(newExpr); + } else { + newOrderBys.add(expr); + } + } + return newOrderBys; + } + + public MySQLAggregate generateAggregate() { + MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); + + if (func.isVariadic()) { + int nrExprs = Randomly.smallNumber() + 1; + List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + + return new MySQLAggregate(exprs, func); + } else { + return new MySQLAggregate(List.of(generateExpression()), func); + } + } + + // --- Shared oracle infrastructure (TLPWhere / CERT / EET) --- + + @Override + public MySQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public MySQLExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public List> generateSetAssignments() { + List> assignments = new ArrayList<>(); + for (MySQLColumn column : Randomly.nonEmptySubset(columns)) { + // As with the normal UPDATE workload, the value is an arbitrary expression (not type-matched to the + // column); + // any resulting type/range error is on the oracle's expected-error allow-list. + assignments.add(new AbstractMap.SimpleEntry<>(column, generateExpression())); + } + return assignments; + } + + @Override + public List generateInsertValues() { + // One value per content column, in schema order (aligned with the INSERT column list). As with the normal + // INSERT workload, each value is an arbitrary expression (not type-matched to the column); any resulting + // type/range/constraint error is on the oracle's expected-error allow-list. + return columns.stream().map(c -> generateExpression()).collect(Collectors.toList()); + } + + @Override + public MySQLSelect generateSelect() { + return new MySQLSelect(); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new MySQLTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); + } + + // --- CERT oracle --- + + @Override + public String generateExplainQuery(MySQLSelect select) { + return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed + // from TRADITIONAL to TREE, hence TRADITIONAL must + // now be specified + } + + @Override + public boolean mutate(MySQLSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateWhere); + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateAnd); + mutators.add(this::mutateOr); + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateDistinct(MySQLSelect select) { + MySQLSelect.SelectType selectType = select.getFromOptions(); + if (selectType != MySQLSelect.SelectType.ALL) { + select.setSelectType(MySQLSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(MySQLSelect.SelectType.DISTINCT); + return false; + } + } + + boolean mutateWhere(MySQLSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression()); + } + return increase; + } + + boolean mutateGroupBy(MySQLSelect select) { + boolean increase = !select.getGroupByExpressions().isEmpty(); + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(MySQLSelect select) { + if (select.getGroupByExpressions().isEmpty()) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(MySQLSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + MySQLBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(MySQLSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + return false; + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + MySQLBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + // --- EET oracle (including DML) --- + + @Override + public EETTransformer createTransformer() { + return new MySQLEETTransformer(this); + } + + // --- EET DML only --- + + @Override + public String asString(MySQLExpression expr) { + return MySQLVisitor.asString(expr); + } + + @Override + public String stampRowIdsStatement(MySQLTable table) { + // MySQL's UUID() gives each existing row a distinct value in a single statement. Stamping happens once, before + // both rolled-back statement runs, so both observe identical identifiers; the standard-SQL statements (add/drop + // column, delete/update, snapshot, transaction control) use EETDMLGenerator's defaults. + return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); + } + + @Override + public String rowIdColumnType() { + // Holds a 36-character UUID string produced by stampRowIdsStatement. + return "VARCHAR(36)"; + } + + @Override + public String insertedRowIdExpression() { + // The source row's identifier with its dashes removed: deterministic (identical across both runs) and unique + // per + // source row. Fits the identifier column's VARCHAR(36). + return String.format("REPLACE(%s, '-', '')", ROW_ID_COLUMN); + } } diff --git a/src/sqlancer/mysql/gen/MySQLHintGenerator.java b/src/sqlancer/mysql/gen/MySQLHintGenerator.java new file mode 100644 index 000000000..dc6138b23 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLHintGenerator.java @@ -0,0 +1,204 @@ +package sqlancer.mysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.mysql.MySQLSchema.MySQLIndex; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLText; + +public class MySQLHintGenerator { + + private final MySQLSelect select; + private final List tables; + private final StringBuilder sb = new StringBuilder(); + + enum OptimizeHint { + BKA, NO_BKA, BNL, NO_BNL, DERIVED_CONDITION_PUSHDOWN, NO_DERIVED_CONDITION_PUSHDOWN, GROUP_INDEX, + NO_GROUP_INDEX, HASH_JOIN, NO_HASH_JOIN, INDEX, NO_INDEX, INDEX_MERGE, NO_INDEX_MERGE, JOIN_FIXED_ORDER, + JOIN_INDEX, NO_JOIN_INDEX, JOIN_ORDER, JOIN_PREFIX, JOIN_SUFFIX, MERGE, NO_MERGE, MRR, NO_MRR, NO_ICP, + NO_RANGE_OPTIMIZATION, ORDER_INDEX, NO_ORDER_INDEX, SEMIJOIN, NO_SEMIJOIN, SKIP_SCAN, NO_SKIP_SCAN + } + + public MySQLHintGenerator(MySQLSelect select, List tables) { + this.select = select; + this.tables = tables; + } + + public static void generateHints(MySQLSelect select, List tables) { + new MySQLHintGenerator(select, tables).randomHint(); + } + + public static List generateAllHints(MySQLSelect select, List tables) { + MySQLHintGenerator generator = new MySQLHintGenerator(select, tables); + return generator.allHints(); + } + + private void randomHint() { + OptimizeHint chosenhint = Randomly.fromOptions(OptimizeHint.values()); + generate(chosenhint); + } + + private List allHints() { + List results = new ArrayList<>(); + for (OptimizeHint hint : OptimizeHint.values()) { + try { + MySQLText generatedHint = generate(hint); + results.add(generatedHint); + } catch (IgnoreMeException e) { + continue; + } + } + return results; + } + + private MySQLText generate(OptimizeHint chosenhint) { + sb.setLength(0); + + switch (chosenhint) { + case BKA: + tablesHint("BKA"); + break; + case NO_BKA: + tablesHint("NO_BKA"); + break; + case BNL: + tablesHint("BNL"); + break; + case NO_BNL: + tablesHint("NO_BNL"); + break; + case DERIVED_CONDITION_PUSHDOWN: + tablesHint("DERIVED_CONDITION_PUSHDOWN"); + break; + case NO_DERIVED_CONDITION_PUSHDOWN: + tablesHint("NO_DERIVED_CONDITION_PUSHDOWN"); + break; + case GROUP_INDEX: + indexesHint("GROUP_INDEX"); + break; + case NO_GROUP_INDEX: + indexesHint("NO_GROUP_INDEX"); + break; + case HASH_JOIN: + tablesHint("HASH_JOIN"); + break; + case NO_HASH_JOIN: + tablesHint("NO_HASH_JOIN"); + break; + case INDEX: + indexesHint("INDEX"); + break; + case NO_INDEX: + indexesHint("NO_INDEX"); + break; + case INDEX_MERGE: + indexesHint("INDEX_MERGE"); + break; + case NO_INDEX_MERGE: + indexesHint("NO_INDEX_MERGE"); + break; + case JOIN_FIXED_ORDER: + tablesHint("JOIN_FIXED_ORDER"); + break; + case JOIN_INDEX: + indexesHint("JOIN_INDEX"); + break; + case NO_JOIN_INDEX: + indexesHint("NO_JOIN_INDEX"); + break; + case JOIN_ORDER: + tablesHint("JOIN_ORDER"); + break; + case JOIN_PREFIX: + tablesHint("JOIN_PREFIX"); + break; + case JOIN_SUFFIX: + tablesHint("JOIN_SUFFIX"); + break; + case MERGE: + tablesHint("MERGE"); + break; + case NO_MERGE: + tablesHint("NO_MERGE"); + break; + case MRR: + indexesHint("MRR"); + break; + case NO_MRR: + indexesHint("NO_MRR"); + break; + case NO_ICP: + indexesHint("NO_ICP"); + break; + case NO_RANGE_OPTIMIZATION: + indexesHint("NO_RANGE_OPTIMIZATION"); + break; + case ORDER_INDEX: + indexesHint("ORDER_INDEX"); + break; + case NO_ORDER_INDEX: + indexesHint("NO_ORDER_INDEX"); + break; + case SEMIJOIN: + semiHint("SEMIJOIN"); + break; + case NO_SEMIJOIN: + semiHint("NO_SEMIJOIN"); + break; + case SKIP_SCAN: + indexesHint("SKIP_SCAN"); + break; + case NO_SKIP_SCAN: + indexesHint("NO_SKIP_SCAN"); + break; + default: + throw new AssertionError(); + } + MySQLText hint = new MySQLText(sb.toString()); + select.setHint(hint); + return hint; + } + + private void indexesHint(String string) { + sb.append(string); + sb.append("("); + MySQLTable table = Randomly.fromList(tables); + List allIndexes = table.getIndexes(); + sb.append(table.getName()); + sb.append(" "); + if (allIndexes.isEmpty()) { + sb.append("PRIMARY"); + } else { + List indexSubset = Randomly.nonEmptySubset(allIndexes); + sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); + } + sb.append(")"); + } + + private void tablesHint(String string) { + sb.append(string); + sb.append("("); + appendTables(); + sb.append(")"); + } + + private void semiHint(String string) { + sb.append(string); + sb.append("("); + String[] options = { "DUPSWEEDOUT", "FIRSTMATCH", "LOOSESCAN", "MATERIALIZATION" }; + List chosenOptions = Randomly.nonEmptySubset(options); + sb.append(chosenOptions.stream().collect(Collectors.joining(", "))); + sb.append(")"); + } + + private void appendTables() { + List tableSubset = Randomly.nonEmptySubset(tables); + sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + } + +} diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 5ab1d3a3b..0e464dead 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -1,42 +1,51 @@ package sqlancer.mysql.gen; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLBugs; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExpression; public class MySQLInsertGenerator { private final MySQLTable table; private final StringBuilder sb = new StringBuilder(); - boolean canFail; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; + private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug + // workarounds) - public MySQLInsertGenerator(MySQLGlobalState globalState) { + public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); + this.table = table; } - public static Query insertRow(MySQLGlobalState globalState) throws SQLException { + public static SQLQueryAdapter insertRow(MySQLGlobalState globalState) throws SQLException { + MySQLTable table = globalState.getSchema().getRandomTable(); + return insertRow(globalState, table); + } + + public static SQLQueryAdapter insertRow(MySQLGlobalState globalState, MySQLTable table) throws SQLException { if (Randomly.getBoolean()) { - return new MySQLInsertGenerator(globalState).generateInsert(); + return new MySQLInsertGenerator(globalState, table).generateInsert(); } else { - return new MySQLInsertGenerator(globalState).generateReplace(); + return new MySQLInsertGenerator(globalState, table).generateReplace(); } } - private Query generateReplace() { - canFail = true; + private SQLQueryAdapter generateReplace() { sb.append("REPLACE"); if (Randomly.getBoolean()) { sb.append(" "); @@ -46,7 +55,7 @@ private Query generateReplace() { } - private Query generateInsert() { + private SQLQueryAdapter generateInsert() { sb.append("INSERT"); if (Randomly.getBoolean()) { sb.append(" "); @@ -54,13 +63,11 @@ private Query generateInsert() { } if (Randomly.getBoolean()) { sb.append(" IGNORE"); - } else { - canFail = true; } return generateInto(); } - private Query generateInto() { + private SQLQueryAdapter generateInto() { sb.append(" INTO "); sb.append(table.getName()); List columns = table.getRandomNonEmptyColumnSubset(); @@ -84,20 +91,64 @@ private Query generateInto() { if (c != 0) { sb.append(", "); } - sb.append(MySQLVisitor.asString(gen.generateConstant())); + MySQLExpression constExpr; + // loop to regenerate until expression is valid (for bug workarounds) + int regenerationAttempts = 0; + while (true) { + regenerationAttempts++; + if (regenerationAttempts > MAX_REGENERATION_ATTEMPTS) { + throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + + " attempts while generating constant for column " + columns.get(c).getName()); + } + constExpr = gen.generateConstant(); + boolean reject = false; + + // Bug workaround: for integer columns, reject values that round to 1 + if (!reject && MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { + if (constExpr instanceof MySQLConstant.MySQLIntConstant) { + long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); + reject = value == 1; + } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { + double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); + reject = value >= 0.5 && value < 1.5; + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may + // be implicitly cast to 1 + reject = true; + } + } + // Bug workaround: for decimal columns, reject values that round to 0 + if (!reject && MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { + if (constExpr instanceof MySQLConstant.MySQLIntConstant) { + long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); + reject = value == 0; + } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { + double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); + reject = value >= -0.5 && value < 0.5; + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may + // be implicitly cast to 0 + reject = true; + } + } + + // Bug workaround: if using CERT oracle, reject NULL values + if (!reject && MySQLBugs.bug120712 + && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.CERT) + && constExpr instanceof MySQLConstant.MySQLNullConstant) { + reject = true; + } + + if (!reject) { + break; + } + } + sb.append(MySQLVisitor.asString(constExpr)); } sb.append(")"); } - errors.add("doesn't have a default value"); - errors.add("Data truncation"); - errors.add("Incorrect integer value"); - errors.add("Duplicate entry"); - errors.add("Data truncated for functional index"); - errors.add("Data truncated for column"); - errors.add("cannot be null"); - errors.add("Incorrect decimal value"); - return new QueryAdapter(sb.toString(), errors); + MySQLErrors.addInsertUpdateErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java new file mode 100644 index 000000000..d701072ee --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -0,0 +1,67 @@ +package sqlancer.mysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableReference; + +public final class MySQLRandomQuerySynthesizer { + + private MySQLRandomQuerySynthesizer() { + } + + public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) { + MySQLTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + MySQLSelect select = new MySQLSelect(); + + List allColumns = new ArrayList<>(); + List columnsWithoutAggregations = new ArrayList<>(); + + boolean hasGeneratedAggregate = false; + + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); + for (int i = 0; i < nrColumns; i++) { + if (Randomly.getBoolean()) { + MySQLExpression expression = gen.generateExpression(); + allColumns.add(expression); + columnsWithoutAggregations.add(expression); + } else { + allColumns.add(gen.generateAggregate()); + hasGeneratedAggregate = true; + } + } + select.setFetchColumns(allColumns); + + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableList); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (hasGeneratedAggregate || Randomly.getBoolean()) { + select.setGroupByExpressions(columnsWithoutAggregations); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBoolean()) { + select.setLimitClause(MySQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(MySQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + return select; + } + +} diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index d00cf27f8..e350685ef 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -1,13 +1,15 @@ package sqlancer.mysql.gen; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.MainOptions; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; public class MySQLSetGenerator { @@ -23,7 +25,7 @@ public MySQLSetGenerator(Randomly r, MainOptions options) { this.isSingleThreaded = options.getNumberConcurrentThreads() == 1; } - public static Query set(MySQLGlobalState globalState) { + public static SQLQueryAdapter set(MySQLGlobalState globalState) { return new MySQLSetGenerator(globalState.getRandomly(), globalState.getOptions()).get(); } @@ -35,11 +37,11 @@ private enum Action { AUTOCOMMIT("autocommit", (r) -> 1, Scope.GLOBAL, Scope.SESSION), // BIG_TABLES("big_tables", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - COMPLETION_TYPE("completion_type", (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", 0, 1, 2), - Scope.GLOBAL), // + COMPLETION_TYPE("completion_type", + (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", "0", "1", "2"), Scope.GLOBAL), // BULK_INSERT_CACHE_SIZE("bulk_insert_buffer_size", (r) -> r.getLong(0, Long.MAX_VALUE), Scope.GLOBAL, // Scope.SESSION), // - CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", 0, 1, 2), // + CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", "0", "1", "2"), // Scope.GLOBAL), // CTE_MAX_RECURSION_DEPTH("cte_max_recursion_depth", // (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), // @@ -67,8 +69,6 @@ private enum Action { MAX_SP_RECURSION_DEPTH("max_sp_recursion_depth", (r) -> r.getLong(0, 255), Scope.GLOBAL, Scope.SESSION), // MYISAM_DATA_POINTER_SIZE("myisam_data_pointer_size", (r) -> r.getLong(2, 7), Scope.GLOBAL), // MYISAM_MAX_SORT_FILE_SIZE("myisam_max_sort_file_size", (r) -> r.getLong(0, 9223372036854775807L), Scope.GLOBAL), // - MYISAM_REPAIR_THREADS("myisam_repair_threads", (r) -> r.getLong(1, Long.MAX_VALUE), Scope.GLOBAL, - Scope.SESSION), // MYISAM_SORT_BUFFER_SIZE("myisam_sort_buffer_size", (r) -> r.getLong(4096, Long.MAX_VALUE), Scope.GLOBAL, Scope.SESSION), // MYISAM_STATS_METHOD("myisam_stats_method", @@ -99,7 +99,6 @@ private enum Action { SCHEMA_DEFINITION_CACHE("schema_definition_cache", (r) -> r.getLong(256, 524288), Scope.GLOBAL), // SHOW_CREATE_TABLE_VERBOSITY("show_create_table_verbosity", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - SHOW_OLD_TEMPORALS("show_old_temporals", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), /* * sort_buffer_size is commented out as a workaround for https://bugs.mysql.com/bug.php?id=95969 */ @@ -128,17 +127,18 @@ private enum Action { this.scopes = scopes.clone(); } - /** + /* * @see https://dev.mysql.com/doc/refman/8.0/en/switchable-optimizations.html */ private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); - String[] options = { "batched_key_access", "block_nested_loop", "condition_fanout_filter", "derived_merge", - "engine_condition_pushdown", "index_condition_pushdown", "use_index_extensions", "index_merge", - "index_merge_intersection", "index_merge_sort_union", "index_merge_union", "use_invisible_indexes", - "mrr", "mrr_cost_based", "skip_scan", "semijoin", "duplicateweedout", "firstmatch", "loosescan", - "materialization", "subquery_materialization_cost_based" }; + String[] options = { "index_merge", "index_merge_union", "index_merge_sort_union", + "index_merge_intersection", "index_condition_pushdown", "mrr", "mrr_cost_based", + "block_nested_loop", "batched_key_access", "materialization", "semijoin", "loosescan", "firstmatch", + "duplicateweedout", "subquery_materialization_cost_based", "use_index_extensions", + "condition_fanout_filter", "derived_merge", "use_invisible_indexes", "skip_scan", "hash_join", + "subquery_to_derived", "prefer_ordering_index", "derived_condition_pushdown" }; List optionSubset = Randomly.nonEmptySubset(options); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); @@ -160,7 +160,7 @@ public Scope[] getScopes() { } } - private Query get() { + private SQLQueryAdapter get() { sb.append("SET "); Action a; if (isSingleThreaded) { @@ -188,7 +188,49 @@ private Query get() { sb.append(a.name); sb.append(" = "); sb.append(a.prod.apply(r)); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); + } + + public static SQLQueryAdapter resetOptimizer() { + return new SQLQueryAdapter("SET optimizer_switch='default'"); + } + + public static List getAllOptimizer(MySQLGlobalState globalState) { + List result = new ArrayList<>(); + String[] options = { "index_merge", "index_merge_union", "index_merge_sort_union", "index_merge_intersection", + "engine_condition_pushdown", "index_condition_pushdown", "mrr", "mrr_cost_based", "block_nested_loop", + "batched_key_access", "materialization", "semijoin", "loosescan", "firstmatch", "duplicateweedout", + "subquery_materialization_cost_based", "use_index_extensions", "condition_fanout_filter", + "derived_merge", "use_invisible_indexes", "skip_scan", "hash_join", "subquery_to_derived", + "prefer_ordering_index", "derived_condition_pushdown" }; + + List availableOptions = new ArrayList<>(Arrays.asList(options)); + if (MySQLBugs.bug112242) { + availableOptions.remove("use_invisible_indexes"); + } + if (MySQLBugs.bug112243) { + availableOptions.remove("subquery_to_derived"); + } + if (MySQLBugs.bug112264) { + availableOptions.remove("block_nested_loop"); + } + + StringBuilder sb = new StringBuilder(); + sb.append("SET "); + if (globalState.getOptions().getNumberConcurrentThreads() == 1 && Randomly.getBoolean()) { + sb.append("GLOBAL"); + } else { + sb.append("SESSION"); + } + sb.append(" optimizer_switch = '%s'"); + + for (String option : availableOptions) { + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=on"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=off"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=default"))); + } + + return result; } } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index c67a805ad..d27d6e681 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -7,42 +7,44 @@ import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; -import sqlancer.sqlite3.gen.SQLite3Common; public class MySQLTableGenerator { - private final StringBuilder sb = new StringBuilder(); private final boolean allowPrimaryKey; private boolean setPrimaryKey; private final String tableName; private final Randomly r; - private int columnId; private boolean tableHasNullableColumn; private MySQLEngine engine; private int keysSpecified; private final List columns = new ArrayList<>(); private final MySQLSchema schema; + private final MySQLGlobalState globalState; - public MySQLTableGenerator(String tableName, Randomly r, MySQLSchema schema) { + public MySQLTableGenerator(MySQLGlobalState globalState, String tableName) { this.tableName = tableName; - this.r = r; - this.schema = schema; + this.r = globalState.getRandomly(); + this.schema = globalState.getSchema(); allowPrimaryKey = Randomly.getBoolean(); + this.globalState = globalState; } - public static Query generate(String tableName, Randomly r, MySQLSchema schema) { - return new MySQLTableGenerator(tableName, r, schema).create(); + public static SQLQueryAdapter generate(MySQLGlobalState globalState, String tableName) { + return new MySQLTableGenerator(globalState, tableName).create(); } - private Query create() { - List errors = new ArrayList<>(); + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); sb.append("CREATE"); // TODO support temporary tables in the schema @@ -50,48 +52,51 @@ private Query create() { if (Randomly.getBoolean()) { sb.append(" IF NOT EXISTS"); } - sb.append(" " + tableName); + sb.append(" "); + sb.append(tableName); if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { sb.append(" LIKE "); sb.append(schema.getRandomTable().getName()); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString(), true); } else { sb.append("("); for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { if (i != 0) { sb.append(", "); } - appendColumn(); + appendColumn(i); } sb.append(")"); sb.append(" "); appendTableOptions(); appendPartitionOptions(); - if ((tableHasNullableColumn || setPrimaryKey) && engine == MySQLEngine.CSV) { + if (engine == MySQLEngine.CSV && (tableHasNullableColumn || setPrimaryKey)) { if (true) { // TODO // results in an error throw new IgnoreMeException(); } - } else if ((tableHasNullableColumn || keysSpecified > 1) && engine == MySQLEngine.ARCHIVE) { + } else if (engine == MySQLEngine.ARCHIVE && (tableHasNullableColumn || keysSpecified > 1)) { errors.add("Too many keys specified; max 1 keys allowed"); errors.add("Table handler doesn't support NULL in given index"); - errors.add("Got error -1 - 'Unknown error -1' from storage engine"); addCommonErrors(errors); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors, true); } addCommonErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } - private void addCommonErrors(List list) { + private void addCommonErrors(ExpectedErrors list) { list.add("The storage engine for the table doesn't support"); list.add("doesn't have this option"); list.add("must include all columns"); list.add("not allowed type for this type of partitioning"); list.add("doesn't support BLOB/TEXT columns"); list.add("A BLOB field is not allowed in partition function"); + list.add("Too many keys specified; max 1 keys allowed"); + list.add("The total length of the partitioning fields is too large"); + list.add("Got error -1 - 'Unknown error -1' from storage engine"); } private enum PartitionOptions { @@ -159,7 +164,15 @@ public static List getRandomTableOptions() { } private void appendTableOptions() { - List tableOptions = TableOptions.getRandomTableOptions(); + List tableOptions = new ArrayList<>(TableOptions.getRandomTableOptions()); + // The EET DML oracle rolls back each statement to compare database states, which requires a transactional + // engine. The ENGINE option already forces InnoDB when the oracle is active (see the ENGINE case below), but it + // is only emitted when randomly chosen; otherwise the table would inherit the server's default engine, which is + // not guaranteed transactional. Force the option to always be present so the engine is never left to the + // server default. + if (globalState.usesEETDML() && !tableOptions.contains(TableOptions.ENGINE)) { + tableOptions.add(TableOptions.ENGINE); + } int i = 0; for (TableOptions o : tableOptions) { if (i++ != 0) { @@ -167,10 +180,13 @@ private void appendTableOptions() { } switch (o) { case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); + sb.append("AUTO_INCREMENT = "); + sb.append(r.getPositiveInteger()); break; + // The valid range for avg_row_length is [0,4294967295] case AVG_ROW_LENGTH: - sb.append("AVG_ROW_LENGTH = " + r.getPositiveInteger()); + sb.append("AVG_ROW_LENGTH = "); + sb.append(r.getLong(0, 4294967295L + 1)); break; case CHECKSUM: sb.append("CHECKSUM = 1"); @@ -190,9 +206,13 @@ private void appendTableOptions() { // "NDB": java.sql.SQLSyntaxErrorException: Unknown storage engine 'NDB' // "EXAMPLE": java.sql.SQLSyntaxErrorException: Unknown storage engine 'EXAMPLE' // "MERGE": java.sql.SQLException: Table 't0' is read only - String fromOptions = Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); + // The EET DML oracle rolls back each statement to compare database states, which requires a + // transactional engine, so only InnoDB is used while it is active. + String fromOptions = globalState.usesEETDML() ? "InnoDB" + : Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); this.engine = MySQLEngine.get(fromOptions); - sb.append("ENGINE = " + fromOptions); + sb.append("ENGINE = "); + sb.append(fromOptions); break; // case ENCRYPTION: // sb.append("ENCRYPTION = '"); @@ -203,27 +223,34 @@ private void appendTableOptions() { sb.append("INSERT_METHOD = "); sb.append(Randomly.fromOptions("NO", "FIRST", "LAST")); break; + // The valid range for key_block_size is [0,65535] case KEY_BLOCK_SIZE: sb.append("KEY_BLOCK_SIZE = "); - sb.append(r.getPositiveInteger()); + sb.append(r.getInteger(0, 65535 + 1)); break; case MAX_ROWS: - sb.append("MAX_ROWS = " + r.getLong(0, Long.MAX_VALUE)); + sb.append("MAX_ROWS = "); + sb.append(r.getLong(0, Long.MAX_VALUE)); break; case MIN_ROWS: - sb.append("MIN_ROWS = " + r.getLong(1, Long.MAX_VALUE)); + sb.append("MIN_ROWS = "); + sb.append(r.getLong(1, Long.MAX_VALUE)); break; case PACK_KEYS: - sb.append("PACK_KEYS = " + Randomly.fromOptions("1", "0", "DEFAULT")); + sb.append("PACK_KEYS = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); break; case STATS_AUTO_RECALC: - sb.append("STATS_AUTO_RECALC = " + Randomly.fromOptions("1", "0", "DEFAULT")); + sb.append("STATS_AUTO_RECALC = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); break; case STATS_PERSISTENT: - sb.append("STATS_PERSISTENT = " + Randomly.fromOptions("1", "0", "DEFAULT")); + sb.append("STATS_PERSISTENT = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); break; case STATS_SAMPLE_PAGES: - sb.append("STATS_SAMPLE_PAGES = " + r.getInteger(1, Short.MAX_VALUE)); + sb.append("STATS_SAMPLE_PAGES = "); + sb.append(r.getInteger(1, Short.MAX_VALUE)); break; default: throw new AssertionError(o); @@ -231,32 +258,21 @@ private void appendTableOptions() { } } - private void appendColumn() { - String columnName = SQLite3Common.createColumnName(columnId); + private void appendColumn(int columnId) { + String columnName = DBMSCommon.createColumnName(columnId); columns.add(columnName); sb.append(columnName); appendColumnDefinition(); - columnId++; } private enum ColumnOptions { NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY } - private void appendColumnDefinition() { - sb.append(" "); - MySQLDataType randomType = MySQLDataType.getRandom(); - boolean isTextType = randomType == MySQLDataType.VARCHAR; - appendTypeString(randomType); - sb.append(" "); - // TODO: this was commented out since it makes the implementation of LIKE more - // difficult - // if (Randomly.getBoolean()) { - // sb.append(" ZEROFILL"); - // } + private void appendColumnOption(MySQLDataType type) { + boolean isTextType = type == MySQLDataType.VARCHAR; boolean isNull = false; boolean columnHasPrimaryKey = false; - List columnOptions = Randomly.subset(ColumnOptions.values()); if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { tableHasNullableColumn = true; @@ -312,10 +328,17 @@ private void appendColumnDefinition() { throw new AssertionError(); } } + } + private void appendColumnDefinition() { + sb.append(" "); + MySQLDataType randomType = MySQLDataType.getRandom(globalState); + appendType(randomType); + sb.append(" "); + appendColumnOption(randomType); } - private void appendTypeString(MySQLDataType randomType) { + private void appendType(MySQLDataType randomType) { switch (randomType) { case DECIMAL: sb.append("DECIMAL"); @@ -326,7 +349,7 @@ private void appendTypeString(MySQLDataType randomType) { if (Randomly.getBoolean()) { sb.append("("); sb.append(Randomly.getNotCachedInteger(0, 255)); // Display width out of range for column 'c0' (max = - // 255) + // 255) sb.append(")"); } break; @@ -335,26 +358,41 @@ private void appendTypeString(MySQLDataType randomType) { break; case FLOAT: sb.append("FLOAT"); - optionallyAddPrecisionAndScale(sb); + optionallyAddFloatingPointPrecisionAndScale(sb); break; case DOUBLE: sb.append(Randomly.fromOptions("DOUBLE", "FLOAT")); - optionallyAddPrecisionAndScale(sb); + optionallyAddFloatingPointPrecisionAndScale(sb); break; default: throw new AssertionError(); } if (randomType.isNumeric()) { - if (Randomly.getBoolean() && randomType != MySQLDataType.INT && !MySQLBugs.bug99127) { + if (Randomly.getBoolean() && randomType != MySQLDataType.INT) { sb.append(" UNSIGNED"); } - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS + || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET + || o == MySQLOracleFactory.EET_DML)) { sb.append(" ZEROFILL"); } } } - public static void optionallyAddPrecisionAndScale(StringBuilder sb) { + // FLOAT(M, D)/DOUBLE(M, D) is deprecated and cannot be reproduced as a CAST target, so the EET oracle's type + // inference relies on FLOAT/DOUBLE columns being created without (M, D) (see MySQLEETTransformer#inferColumnType); + // it is therefore omitted while EET is active. DECIMAL(M, D) has no such restriction: the EET oracle tracks its + // (M, D) and reproduces it via CAST(... AS DECIMAL(M, D)), so it keeps using optionallyAddPrecisionAndScale. + private void optionallyAddFloatingPointPrecisionAndScale(StringBuilder sb) { + // Both EET oracles rely on the same type inference (MySQLEETTransformer), so both omit FLOAT(M, D)/DOUBLE(M, + // D). + if (!globalState.usesEET()) { + optionallyAddPrecisionAndScale(sb); + } + } + + private void optionallyAddPrecisionAndScale(StringBuilder sb) { if (Randomly.getBoolean() && !MySQLBugs.bug99183) { sb.append("("); // The maximum number of digits (M) for DECIMAL is 65 diff --git a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java index 808622b29..58b6958c1 100644 --- a/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTruncateTableGenerator.java @@ -1,9 +1,7 @@ package sqlancer.mysql.gen; -import java.util.Arrays; - -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; public final class MySQLTruncateTableGenerator { @@ -11,10 +9,10 @@ public final class MySQLTruncateTableGenerator { private MySQLTruncateTableGenerator() { } - public static Query generate(MySQLGlobalState globalState) { + public static SQLQueryAdapter generate(MySQLGlobalState globalState) { StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); sb.append(globalState.getSchema().getRandomTable().getName()); - return new QueryAdapter(sb.toString(), Arrays.asList("doesn't have this option")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("doesn't have this option")); } } diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java new file mode 100644 index 000000000..ad13a148a --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -0,0 +1,55 @@ +package sqlancer.mysql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; + +public class MySQLUpdateGenerator extends AbstractUpdateGenerator { + + private final MySQLGlobalState globalState; + private MySQLExpressionGenerator gen; + + public MySQLUpdateGenerator(MySQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter create(MySQLGlobalState globalState) { + return new MySQLUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + MySQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new MySQLExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + if (Randomly.getBoolean()) { + MySQLErrors.addExpressionErrors(errors); + appendWhereClause(MySQLVisitor.asString(gen.generateExpression())); + } + MySQLErrors.addInsertUpdateErrors(errors); + errors.add("doesn't have this option"); + } + + @Override + protected void updateValue(MySQLColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant()); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append(MySQLVisitor.asString(gen.generateExpression())); + } + } + +} diff --git a/src/sqlancer/mysql/gen/admin/MySQLFlush.java b/src/sqlancer/mysql/gen/admin/MySQLFlush.java index ee3699528..2558dbc7a 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLFlush.java +++ b/src/sqlancer/mysql/gen/admin/MySQLFlush.java @@ -3,13 +3,12 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; -/** +/* * https://dev.mysql.com/doc/refman/8.0/en/flush.html#flush-tables-variants */ public class MySQLFlush { @@ -21,11 +20,11 @@ public MySQLFlush(List tables) { this.tables = tables; } - public static Query create(MySQLGlobalState globalState) { + public static SQLQueryAdapter create(MySQLGlobalState globalState) { return new MySQLFlush(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty()).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { sb.append("FLUSH"); if (Randomly.getBoolean()) { sb.append(" "); @@ -45,7 +44,7 @@ private Query generate() { // TODO implement READ LOCK and other variants } } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/admin/MySQLReset.java b/src/sqlancer/mysql/gen/admin/MySQLReset.java index a210b018e..dedb34a21 100644 --- a/src/sqlancer/mysql/gen/admin/MySQLReset.java +++ b/src/sqlancer/mysql/gen/admin/MySQLReset.java @@ -2,9 +2,8 @@ import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; public final class MySQLReset { @@ -12,11 +11,11 @@ public final class MySQLReset { private MySQLReset() { } - public static Query create(MySQLGlobalState globalState) { + public static SQLQueryAdapter create(MySQLGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("RESET "); sb.append(Randomly.nonEmptySubset("MASTER", "SLAVE").stream().collect(Collectors.joining(", "))); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index e935e7d9c..028886831 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -1,12 +1,11 @@ package sqlancer.mysql.gen.datadef; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -33,12 +32,12 @@ public MySQLIndexGenerator(MySQLSchema schema, Randomly r, MySQLGlobalState glob this.globalState = globalState; } - public static Query create(MySQLGlobalState globalState) { + public static SQLQueryAdapter create(MySQLGlobalState globalState) { return new MySQLIndexGenerator(globalState.getSchema(), globalState.getRandomly(), globalState).create(); } - public Query create() { - Set errors = new HashSet<>(); + public SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); MySQLErrors.addExpressionErrors(errors); sb.append("CREATE "); if (Randomly.getBoolean()) { @@ -80,7 +79,11 @@ public Query create() { if (Randomly.getBoolean() && c.getType() == MySQLDataType.VARCHAR) { sb.append("("); // TODO for string - sb.append(r.getInteger(1, 5)); + if (MySQLBugs.bug114534) { + sb.append(r.getInteger(2, 5)); + } else { + sb.append(r.getInteger(1, 5)); + } sb.append(")"); } if (Randomly.getBoolean()) { @@ -116,7 +119,10 @@ public Query create() { errors.add("out of range"); errors.add("Data truncated for functional index"); errors.add("used in key specification without a key length"); - return new QueryAdapter(string, errors, true); + errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration + errors.add("in the PARTITION BY KEY() clause is not supported"); // prefix key parts disallowed on + // KEY-partitioned columns + return new SQLQueryAdapter(string, errors, true); } private void algorithmOption() { diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java index 6f34395fb..c7533b0c7 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLAnalyzeTable.java @@ -3,15 +3,14 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/analyze-table.html + * @see ANALYZE TABLE Statement */ public class MySQLAnalyzeTable { @@ -24,12 +23,12 @@ public MySQLAnalyzeTable(List tables, Randomly r) { this.r = r; } - public static Query analyze(MySQLGlobalState globalState) { + public static SQLQueryAdapter analyze(MySQLGlobalState globalState) { return new MySQLAnalyzeTable(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty(), globalState.getRandomly()).generate(); } - private Query generate() { + private SQLQueryAdapter generate() { sb.append("ANALYZE "); if (Randomly.getBoolean()) { sb.append(Randomly.fromOptions("NO_WRITE_TO_BINLOG", "LOCAL")); @@ -44,7 +43,7 @@ private Query generate() { updateHistogram(); } } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } // ANALYZE [NO_WRITE_TO_BINLOG | LOCAL] diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java index 6b1e5df1e..86e950cf0 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLCheckTable.java @@ -3,14 +3,13 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/check-table.html + * @see CHECK TABLE Statement */ public class MySQLCheckTable { @@ -21,7 +20,7 @@ public MySQLCheckTable(List tables) { this.tables = tables; } - public static Query check(MySQLGlobalState globalState) { + public static SQLQueryAdapter check(MySQLGlobalState globalState) { return new MySQLCheckTable(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty()).generate(); } @@ -35,13 +34,13 @@ public static Query check(MySQLGlobalState globalState) { // | EXTENDED // | CHANGED // } - private Query generate() { + private SQLQueryAdapter generate() { sb.append("CHECK TABLE "); sb.append(tables.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); sb.append(" "); List options = Randomly.subset("FOR UPGRADE", "QUICK", "FAST", "MEDIUM", "EXTENDED", "CHANGED"); sb.append(options.stream().collect(Collectors.joining(" "))); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java index 55030a5bc..a9442c003 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLChecksum.java @@ -3,14 +3,13 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/checksum-table.html + * @see CHECKSUM TABLE Statement */ public class MySQLChecksum { @@ -21,19 +20,19 @@ public MySQLChecksum(List tables) { this.tables = tables; } - public static Query checksum(MySQLGlobalState globalState) { + public static SQLQueryAdapter checksum(MySQLGlobalState globalState) { return new MySQLChecksum(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty()).checksum(); } // CHECKSUM TABLE tbl_name [, tbl_name] ... [QUICK | EXTENDED] - private Query checksum() { + private SQLQueryAdapter checksum() { sb.append("CHECKSUM TABLE "); sb.append(tables.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); if (Randomly.getBoolean()) { sb.append(" "); sb.append(Randomly.fromOptions("QUICK", "EXTENDED")); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java index 73355ef7b..2ee0fe6d2 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLOptimize.java @@ -3,14 +3,13 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/optimize-table.html + * @see OPTIMIZE TABLE Statement */ public class MySQLOptimize { @@ -21,13 +20,13 @@ public MySQLOptimize(List tables) { this.tables = tables; } - public static Query optimize(MySQLGlobalState globalState) { + public static SQLQueryAdapter optimize(MySQLGlobalState globalState) { return new MySQLOptimize(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty()).optimize(); } // OPTIMIZE [NO_WRITE_TO_BINLOG | LOCAL] // TABLE tbl_name [, tbl_name] ... - private Query optimize() { + private SQLQueryAdapter optimize() { sb.append("OPTIMIZE"); if (Randomly.getBoolean()) { sb.append(" "); @@ -35,7 +34,7 @@ private Query optimize() { } sb.append(" TABLE "); sb.append(tables.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java index 5f8171961..322fa353e 100644 --- a/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java +++ b/src/sqlancer/mysql/gen/tblmaintenance/MySQLRepair.java @@ -3,15 +3,14 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; /** - * @see https://dev.mysql.com/doc/refman/8.0/en/repair-table.html + * @see REPAIR TABLE Statement */ public class MySQLRepair { @@ -22,12 +21,12 @@ public MySQLRepair(List tables) { this.tables = tables; } - public static Query repair(MySQLGlobalState globalState) { + public static SQLQueryAdapter repair(MySQLGlobalState globalState) { List tables = globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty(); for (MySQLTable table : tables) { // see https://bugs.mysql.com/bug.php?id=95820 if (table.getEngine() == MySQLEngine.MY_ISAM) { - return new QueryAdapter("SELECT 1"); + return new SQLQueryAdapter("SELECT 1"); } } return new MySQLRepair(tables).repair(); @@ -36,7 +35,7 @@ public static Query repair(MySQLGlobalState globalState) { // REPAIR [NO_WRITE_TO_BINLOG | LOCAL] // TABLE tbl_name [, tbl_name] ... // [QUICK] [EXTENDED] [USE_FRM] - private Query repair() { + private SQLQueryAdapter repair() { sb.append("REPAIR"); if (Randomly.getBoolean()) { sb.append(" "); @@ -53,7 +52,7 @@ private Query repair() { if (Randomly.getBoolean()) { sb.append(" USE_FRM"); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java new file mode 100644 index 000000000..429ef3c89 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -0,0 +1,554 @@ +package sqlancer.mysql.oracle; + +import static sqlancer.ComparatorHelper.getResultSetFirstColumnAsString; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.beust.jcommander.Strings; + +import sqlancer.Randomly; +import sqlancer.common.oracle.DQEBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryError; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTables; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + +public class MySQLDQEOracle extends DQEBase implements TestOracle { + + private final MySQLSchema schema; + private static final String APPEND_ORDER_BY = "%s ORDER BY %s"; + private static final String APPEND_LIMIT = "%s LIMIT %d"; + private final List orderColumns = new ArrayList<>(); + private boolean generateLimit; + private boolean generateOrderBy; + private boolean operateOnSingleTable; + private int limit; + + public MySQLDQEOracle(MySQLGlobalState state) { + super(state); + schema = state.getSchema(); + + MySQLErrors.addExpressionErrors(selectExpectedErrors); + + MySQLErrors.addExpressionErrors(updateExpectedErrors); + MySQLErrors.addInsertUpdateErrors(updateExpectedErrors); + + MySQLErrors.addExpressionErrors(deleteExpectedErrors); + deleteExpectedErrors.add("a foreign key constraint fails"); + } + + @Override + public String generateSelectStatement(AbstractTables tables, String tableName, String whereClauseStr) { + operateOnSingleTable = tables.getTables().size() == 1; + List selectColumns = new ArrayList<>(); + MySQLTables mySQLTables = (MySQLTables) tables; + for (MySQLTable table : mySQLTables.getTables()) { + selectColumns.add(table.getName() + "." + COLUMN_ROWID); + } + if (operateOnSingleTable && Randomly.getBooleanWithSmallProbability()) { + generateOrderBy = true; + // generate order by columns + for (MySQLColumn column : Randomly.nonEmptySubset(mySQLTables.getColumns())) { + orderColumns.add(column.getFullQualifiedName()); + } + // rowId tiebreaker ensures ORDER BY LIMIT is deterministic when user columns have duplicate values + for (MySQLTable table : mySQLTables.getTables()) { + orderColumns.add(table.getName() + "." + COLUMN_ROWID); + } + + if (Randomly.getBooleanWithRatherLowProbability()) { + generateLimit = true; + limit = (int) Randomly.getNotCachedInteger(1, 10); + } + } + + String selectStmt = String.format("SELECT %s FROM %s WHERE %s", Strings.join(",", selectColumns).toLowerCase(), + tableName, whereClauseStr); + if (generateOrderBy) { + selectStmt = String.format(APPEND_ORDER_BY, selectStmt, String.join(",", orderColumns)); + if (generateLimit) { + selectStmt = String.format(APPEND_LIMIT, selectStmt, limit); + } + } + return selectStmt; + } + + @Override + public String generateUpdateStatement(AbstractTables tables, String tableName, String whereClauseStr) { + List updateColumns = new ArrayList<>(); + MySQLTables mySQLTables = (MySQLTables) tables; + for (MySQLTable table : mySQLTables.getTables()) { + updateColumns.add(String.format("%s = 1", table.getName() + "." + COLUMN_UPDATED)); + } + String updateStmt = String.format("UPDATE %s SET %s WHERE %s", tableName, Strings.join(",", updateColumns), + whereClauseStr); + if (generateOrderBy) { + updateStmt = String.format(APPEND_ORDER_BY, updateStmt, String.join(",", orderColumns)); + if (generateLimit) { + updateStmt = String.format(APPEND_LIMIT, updateStmt, limit); + } + } + return updateStmt; + } + + @Override + public String generateDeleteStatement(String tableName, String whereClauseStr) { + String deleteStmt; + if (operateOnSingleTable) { + deleteStmt = String.format("DELETE FROM %s WHERE %s", tableName, whereClauseStr); + if (generateOrderBy) { + deleteStmt = String.format(APPEND_ORDER_BY, deleteStmt, String.join(",", orderColumns)); + if (generateLimit) { + deleteStmt = String.format(APPEND_LIMIT, deleteStmt, limit); + } + } + } else { + deleteStmt = String.format("DELETE %s FROM %s WHERE %s", tableName, tableName, whereClauseStr); + } + return deleteStmt; + } + + @Override + public void check() throws SQLException { + + MySQLTables tables = schema.getRandomTableNonEmptyTables(); + String tableName = tables.getTables().stream().map(AbstractTable::getName).collect(Collectors.joining(",")); + + // DQE does not support aggregate functions, windows functions + // This method does not generate them, may need some configurations if they can be generated + MySQLExpressionGenerator expressionGenerator = new MySQLExpressionGenerator(state) + .setColumns(tables.getColumns()); + MySQLExpression whereClause = expressionGenerator.generateExpression(); + + // MySQLVisitor is not deterministic, we should keep it only once. + // Especially, in MySQLUnaryPostfixOperation and MySQLUnaryPrefixOperation + String whereClauseStr = MySQLVisitor.asString(whereClause); + + String selectStmt = generateSelectStatement(tables, tableName, whereClauseStr); + + String updateStmt = generateUpdateStatement(tables, tableName, whereClauseStr); + + String deleteStmt = generateDeleteStatement(tableName, whereClauseStr); + + for (MySQLTable table : tables.getTables()) { + addAuxiliaryColumns(table); + } + + state.getState().getLocalState().log(selectStmt); + SQLQueryResult selectExecutionResult = executeSelect(selectStmt, tables); + state.getState().getLocalState().log(selectExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(selectExecutionResult.getQueryErrors().toString()); + + state.getState().getLocalState().log(updateStmt); + SQLQueryResult updateExecutionResult = executeUpdate(updateStmt, tables); + state.getState().getLocalState().log(updateExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(updateExecutionResult.getQueryErrors().toString()); + + state.getState().getLocalState().log(deleteStmt); + SQLQueryResult deleteExecutionResult = executeDelete(deleteStmt, tables); + state.getState().getLocalState().log(deleteExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(deleteExecutionResult.getQueryErrors().toString()); + + String compareSelectAndUpdate = compareSelectAndUpdate(selectExecutionResult, updateExecutionResult); + String compareSelectAndDelete = compareSelectAndDelete(selectExecutionResult, deleteExecutionResult); + String compareUpdateAndDelete = compareUpdateAndDelete(updateExecutionResult, deleteExecutionResult); + + String errorMessage = compareSelectAndUpdate == null ? "" : compareSelectAndUpdate + "\n"; + errorMessage += compareSelectAndDelete == null ? "" : compareSelectAndDelete + "\n"; + errorMessage += compareUpdateAndDelete == null ? "" : compareUpdateAndDelete + "\n"; + + if (!errorMessage.isEmpty()) { + throw new AssertionError(errorMessage); + } + + for (MySQLTable table : tables.getTables()) { + dropAuxiliaryColumns(table); + } + } + + public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult updateResult) { + if (updateResult.hasEmptyErrors()) { + if (!selectResult.hasEmptyErrors()) { + // Tolerate SELECT-only discrepancy errors (e.g. 1292 raised in SELECT but not UPDATE + // due to different short-circuit evaluation paths). + boolean selectHasNonDiscrepancyErrors = selectResult.getQueryErrors().stream() + .anyMatch(e -> !isKnownSelectDMLDiscrepancy(e)); + if (selectHasNonDiscrepancyErrors) { + return "SELECT has errors, but UPDATE does not."; + } + } + if (!selectResult.hasSameAccessedRows(updateResult)) { + return "SELECT accessed different rows from UPDATE."; + } + } else { // update has errors + if (hasUpdateSpecificErrors(updateResult)) { + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when specific errors happen."; + } else { + // we do not compare update with select when update has specific errors + return null; + } + } + + // update errors should all appear in the select errors + // known SELECT/DML discrepancy errors are skipped: see KnownSelectDMLDiscrepancy for the full list. + List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); + for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { + SQLQueryError updateError = updateResult.getQueryErrors().get(i); + if (isKnownSelectDMLDiscrepancy(updateError)) { + continue; + } + if (!isFound(selectErrors, updateError)) { + return "SELECT has different errors from UPDATE."; + } + } + + if (hasStopErrors(updateResult)) { + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when stop errors happen."; + } + } else { + if (!selectResult.hasSameAccessedRows(updateResult)) { + return "SELECT accessed different rows from UPDATE when errors happen."; + } + } + } + return null; + } + + /** + * + * @param selectErrors + * selectQueryErrors + * @param targetError + * update or delete queryError + * + * @return is targetError found in selectQueryErrors + */ + private static boolean isFound(List selectErrors, SQLQueryError targetError) { + boolean found = false; + for (int i = 0; i < selectErrors.size(); i++) { + SQLQueryError selectError = selectErrors.get(i); + if (selectError.hasSameCodeAndMessage(targetError)) { + selectErrors.remove(i); + found = true; + break; + } + } + return found; + } + + public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { + if (deleteResult.hasEmptyErrors()) { + if (!selectResult.hasEmptyErrors()) { + // Tolerate SELECT-only discrepancy errors (e.g. 1292 raised in SELECT but not DELETE + // due to different short-circuit evaluation paths). + boolean selectHasNonDiscrepancyErrors = selectResult.getQueryErrors().stream() + .anyMatch(e -> !isKnownSelectDMLDiscrepancy(e)); + if (selectHasNonDiscrepancyErrors) { + return "SELECT has errors, but DELETE does not."; + } + } + if (!selectResult.hasSameAccessedRows(deleteResult)) { + return "SELECT accessed different rows from DELETE."; + } + } else { // delete has errors + if (hasDeleteSpecificErrors(deleteResult)) { + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when specific errors happen."; + } else { + // we do not compare delete with select when delete has specific errors + return null; + } + } + + // delete errors should all appear in the select errors + // known SELECT/DML discrepancy errors are skipped: see KnownSelectDMLDiscrepancy for the full list. + List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); + for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { + SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); + if (isKnownSelectDMLDiscrepancy(deleteError)) { + continue; + } + if (!isFound(selectErrors, deleteError)) { + return "SELECT has different errors from DELETE."; + } + } + + if (hasStopErrors(deleteResult)) { + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when stop errors happen."; + } + } else { + if (!selectResult.hasSameAccessedRows(deleteResult)) { + return "SELECT accessed different rows from DELETE when errors happen."; + } + } + } + return null; + } + + public String compareUpdateAndDelete(SQLQueryResult updateResult, SQLQueryResult deleteResult) { + if (updateResult.hasEmptyErrors() && deleteResult.hasEmptyErrors()) { + if (updateResult.hasSameAccessedRows(deleteResult)) { + return null; + } else { + return "UPDATE accessed different rows from DELETE."; + } + } else { // update or delete has errors + boolean hasSpecificErrors = false; + + if (hasUpdateSpecificErrors(updateResult)) { + hasSpecificErrors = true; + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when specific errors happen."; + } + } + + if (hasDeleteSpecificErrors(deleteResult)) { + hasSpecificErrors = true; + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when specific errors happen."; + } + } + + // when one of these statements has specific errors, do not compare them + if (hasSpecificErrors) { + return null; + } + + if (!updateResult.hasSameErrors(deleteResult)) { + return "UPDATE has different errors from DELETE."; + } else { + if (!hasStopErrors(updateResult)) { + if (!updateResult.hasSameAccessedRows(deleteResult)) { + return "UPDATE accessed different rows from DELETE."; + } + } else { + if (updateResult.hasAccessedRows() || deleteResult.hasAccessedRows()) { + return "UPDATE or DELETE accessed non-empty rows when stop errors happen."; + } + } + } + + return null; + } + } + + /* + * when update violates column constraints, such as not null, unique, primary key and generated column, we cannot + * compare it with other queries. + */ + private boolean hasUpdateSpecificErrors(SQLQueryResult updateResult) { + return updateResult.getQueryErrors().stream().anyMatch( + error -> new MySQLErrorCodeStrategy().getUpdateSpecificErrorCodes().contains(error.getCode())); + } + + /* + * when delete violates column constraints, such as foreign key, we cannot compare it with other queries. + */ + private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { + return deleteResult.getQueryErrors().stream().anyMatch( + error -> new MySQLErrorCodeStrategy().getDeleteSpecificErrorCodes().contains(error.getCode())); + + } + + // Errors MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. Acceptable + // discrepancies that should be skipped; not treated as stop errors so row comparison proceeds normally. + private enum KnownSelectDMLDiscrepancy { + // WHERE clause type coercion: MySQL may short-circuit in SELECT but evaluate fully in UPDATE/DELETE, + // raising this at ERROR level vs WARNING in SELECT. + TRUNCATED_DOUBLE_VALUE(1292), + // Same WHERE clause coercion discrepancy as TRUNCATED_DOUBLE_VALUE. + INCORRECT_COLUMN_VALUE(1366), + // Raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes. + STORAGE_ENGINE_ERROR(1030), + // MySQL applies this memory budget differently for SELECT vs DML; the fallback full-scan still + // evaluates the WHERE predicate correctly. + RANGE_OPTIMIZER_MEM_EXCEEDED(3170), + // Raised when a functional index expression truncates a value during DML; structurally impossible in SELECT. + FUNCTIONAL_INDEX_DATA_TRUNCATED(3751); + + private final int code; + + KnownSelectDMLDiscrepancy(int code) { + this.code = code; + } + } + + private static boolean isKnownSelectDMLDiscrepancy(SQLQueryError error) { + for (KnownSelectDMLDiscrepancy discrepancy : KnownSelectDMLDiscrepancy.values()) { + if (discrepancy.code == error.getCode()) { + return true; + } + } + return false; + } + + private boolean hasStopErrors(SQLQueryResult queryResult) { + return queryResult.getQueryErrors().stream() + .anyMatch(error -> error.getLevel() == SQLQueryError.ErrorLevel.ERROR); + } + + private SQLQueryResult executeSelect(String selectStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + SQLancerResultSet resultSet = null; + try { + resultSet = new SQLQueryAdapter(selectStmt, selectExpectedErrors).executeAndGet(state, false); + } catch (SQLException ignored) { + // we ignore this error, and use get errors to catch it + } finally { + queryErrors = getErrors(); + + if (resultSet != null) { + for (MySQLTable table : tables.getTables()) { + HashSet rows = new HashSet<>(); + accessedRows.put(table, rows); + } + while (resultSet.next()) { + for (MySQLTable table : tables.getTables()) { + accessedRows.get(table).add(resultSet.getString(table.getName() + "." + COLUMN_ROWID)); + } + } + resultSet.close(); + } + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private SQLQueryResult executeUpdate(String updateStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + try { + new SQLQueryAdapter("BEGIN").execute(state, false); + new SQLQueryAdapter(updateStmt, updateExpectedErrors).execute(state, false); + } catch (SQLException ignored) { + // we ignore this error, and we use get errors to catch it + } finally { + queryErrors = getErrors(); + + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String updated = tableName + "." + COLUMN_UPDATED; + String selectRowIdWithUpdated = String.format("SELECT %s FROM %s WHERE %s = 1", rowId, tableName, + updated); + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowIdWithUpdated, updateExpectedErrors, state)); + accessedRows.put(table, rows); + } + + new SQLQueryAdapter("ROLLBACK").execute(state, false); + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private SQLQueryResult executeDelete(String deleteStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + try { + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowId, deleteExpectedErrors, state)); + accessedRows.put(table, rows); + } + + new SQLQueryAdapter("BEGIN").execute(state, false); + new SQLQueryAdapter(deleteStmt, deleteExpectedErrors).execute(state, false); + } catch (SQLException ignored) { + // we ignore this error, and use get errors to catch it + } finally { + queryErrors = getErrors(); + + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowId, deleteExpectedErrors, state)); + accessedRows.get(table).removeAll(rows); + } + + new SQLQueryAdapter("ROLLBACK").execute(state, false); + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private List getErrors() throws SQLException { + SQLancerResultSet resultSet = new SQLQueryAdapter("SHOW WARNINGS").executeAndGet(state, false); + List queryErrors = new ArrayList<>(); + if (resultSet != null) { + while (resultSet.next()) { + SQLQueryError queryError = new SQLQueryError(); + queryError.setLevel(resultSet.getString("Level").equalsIgnoreCase("ERROR") + ? SQLQueryError.ErrorLevel.ERROR : SQLQueryError.ErrorLevel.WARNING); + queryError.setCode(resultSet.getInt("Code")); + queryError.setMessage(resultSet.getString("Message")); + queryErrors.add(queryError); + } + resultSet.close(); + } + Collections.sort(queryErrors); + return queryErrors; + } + + @Override + public void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { + String tableName = table.getName(); + + String addColumnRowID = String.format("ALTER TABLE %s ADD %s VARCHAR(36)", tableName, COLUMN_ROWID); + new SQLQueryAdapter(addColumnRowID).execute(state, false); + state.getState().getLocalState().log(addColumnRowID); + + String addColumnUpdated = String.format("ALTER TABLE %s ADD %s INT DEFAULT 0", tableName, COLUMN_UPDATED); + new SQLQueryAdapter(addColumnUpdated).execute(state, false); + state.getState().getLocalState().log(addColumnUpdated); + + String updateRowsWithUniqueID = String.format("UPDATE %s SET %s = UUID()", tableName, COLUMN_ROWID); + new SQLQueryAdapter(updateRowsWithUniqueID).execute(state, false); + state.getState().getLocalState().log(updateRowsWithUniqueID); + } + + public static class MySQLErrorCodeStrategy implements ErrorCodeStrategy { + @Override + public Set getUpdateSpecificErrorCodes() { + // 1048, Column 'c0' cannot be null + // 1062, Duplicate entry '2' for key 't1.i0 + // 3105, The value specified for generated column 'c1' in table 't1' is not allowed + return Set.of(1048, 1062, 3105); + } + + @Override + public Set getDeleteSpecificErrorCodes() { + // 1451, Cannot delete or update a parent row: a foreign key constraint fails + return Set.of(1451); + } + } +} diff --git a/src/sqlancer/mysql/oracle/MySQLDQPOracle.java b/src/sqlancer/mysql/oracle/MySQLDQPOracle.java new file mode 100644 index 000000000..414ffb156 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLDQPOracle.java @@ -0,0 +1,104 @@ +package sqlancer.mysql.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLJoin; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; +import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.gen.MySQLHintGenerator; +import sqlancer.mysql.gen.MySQLSetGenerator; + +public class MySQLDQPOracle implements TestOracle { + private final MySQLGlobalState state; + private MySQLExpressionGenerator gen; + private MySQLSelect select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public MySQLDQPOracle(MySQLGlobalState globalState) { + state = globalState; + MySQLErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + // Randomly generate a query + MySQLTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new MySQLExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList())); + + select = new MySQLSelect(); + select.setFetchColumns(fetchColumns); + + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateExpression()); + } + } + + // Set the join. + List joinExpressions = MySQLJoin.getRandomJoinClauses(tables.getTables(), state); + select.setJoinList(joinExpressions.stream().map(j -> (MySQLExpression) j).collect(Collectors.toList())); + + // Set the from clause from the tables that are not used in the join. + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableList); + + // Get the result of the first query + String originalQueryString = MySQLVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + // Check hints + List hintList = MySQLHintGenerator.generateAllHints(select, tables.getTables()); + for (MySQLText hint : hintList) { + select.setHint(hint); + String queryString = MySQLVisitor.asString(select); + List result = ComparatorHelper.getResultSetFirstColumnAsString(queryString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, List.of(queryString), + state); + } + + // Check optimizer variables + List optimizationList = MySQLSetGenerator.getAllOptimizer(state); + for (SQLQueryAdapter optimization : optimizationList) { + optimization.execute(state); + List result = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, + List.of(originalQueryString), state); + } catch (AssertionError e) { + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + originalResult.size(), result.size(), originalQueryString, originalResult.size(), + String.join(";", originalQueryString), result.size()); + assertionMessage += System.lineSeparator() + "The setting: " + optimization.getQueryString(); + throw new AssertionError(assertionMessage); + } + } + } +} diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java new file mode 100644 index 000000000..22c0302df --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -0,0 +1,292 @@ +package sqlancer.mysql.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.oracle.EETTransformer; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLBetweenOperation; +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; +import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLCastOperation; +import sqlancer.mysql.ast.MySQLCastOperation.CastType; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLComputableFunction; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExists; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + +/** + * MySQL implementation of the {@link EETTransformer EET} tree-walker. Implements {@link #descend} to rebuild MySQL AST + * nodes from their transformed children, threading the correct boolean/scalar context into each child. + * + *

+ * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target + * types ({@link CastType}, which carries {@code (M, D)} for DECIMAL): {@link #inferType} conservatively classifies AST + * nodes into that domain (returning {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type + * of a random expression by wrapping it in a CAST. + */ +public class MySQLEETTransformer extends EETTransformer { + + private static final boolean BOOLEAN = true; + private static final boolean SCALAR = false; + + private final MySQLExpressionGenerator gen; + + public MySQLEETTransformer(MySQLExpressionGenerator gen) { + this.gen = gen; + } + + @Override + protected MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { + if (expr instanceof MySQLBinaryLogicalOperation) { + // AND/OR/XOR: both operands are evaluated in a boolean context. + MySQLBinaryLogicalOperation op = (MySQLBinaryLogicalOperation) expr; + return new MySQLBinaryLogicalOperation(transformNode(op.getLeft(), BOOLEAN, false), + transformNode(op.getRight(), BOOLEAN, false), op.getOp()); + } else if (expr instanceof MySQLBinaryComparisonOperation) { + MySQLBinaryComparisonOperation op = (MySQLBinaryComparisonOperation) expr; + return new MySQLBinaryComparisonOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLBinaryOperation) { + MySQLBinaryOperation op = (MySQLBinaryOperation) expr; + return new MySQLBinaryOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + MySQLUnaryPrefixOperation op = (MySQLUnaryPrefixOperation) expr; + boolean childContext = op.getOp() == MySQLUnaryPrefixOperator.NOT ? BOOLEAN : SCALAR; + return new MySQLUnaryPrefixOperation(transformNode(op.getExpression(), childContext, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPostfixOperation) { + // The operand is transformed value-preservingly (scalar), which is safe for IS NULL/IS TRUE/IS FALSE. + MySQLUnaryPostfixOperation op = (MySQLUnaryPostfixOperation) expr; + return new MySQLUnaryPostfixOperation(transformNode(op.getExpression(), SCALAR, false), op.getOperator(), + op.isNegated()); + } else if (expr instanceof MySQLCastOperation) { + MySQLCastOperation op = (MySQLCastOperation) expr; + return new MySQLCastOperation(transformNode(op.getExpr(), SCALAR, false), op.getType()); + } else if (expr instanceof MySQLBetweenOperation) { + MySQLBetweenOperation op = (MySQLBetweenOperation) expr; + return new MySQLBetweenOperation(transformNode(op.getExpr(), SCALAR, false), + transformNode(op.getLeft(), SCALAR, false), transformNode(op.getRight(), SCALAR, false)); + } else if (expr instanceof MySQLInOperation) { + MySQLInOperation op = (MySQLInOperation) expr; + List listElements = op.getListElements().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); + return new MySQLInOperation(transformNode(op.getExpr(), SCALAR, false), listElements, op.isTrue()); + } else if (expr instanceof MySQLComputableFunction) { + MySQLComputableFunction op = (MySQLComputableFunction) expr; + MySQLExpression[] args = op.getArguments(); + MySQLExpression[] newArgs = new MySQLExpression[args.length]; + for (int i = 0; i < args.length; i++) { + newArgs[i] = transformNode(args[i], SCALAR, false); + } + return new MySQLComputableFunction(op.getFunction(), newArgs); + } else if (expr instanceof MySQLCaseOperator) { + return descendCase((MySQLCaseOperator) expr); + } + return expr; + } + + private MySQLExpression descendCase(MySQLCaseOperator caseOp) { + MySQLExpression switchCondition = caseOp.getSwitchCondition(); + // Without a switch operand the WHEN conditions are boolean; with one they are compared against the operand. + boolean conditionContext = switchCondition == null ? BOOLEAN : SCALAR; + MySQLExpression newSwitch = switchCondition == null ? null : transformNode(switchCondition, SCALAR, false); + List conditions = caseOp.getConditions().stream() + .map(e -> transformNode(e, conditionContext, false)).collect(Collectors.toList()); + List expressions = caseOp.getExpressions().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); + MySQLExpression elseExpr = caseOp.getElseExpr() == null ? null + : transformNode(caseOp.getElseExpr(), SCALAR, false); + return new MySQLCaseOperator(newSwitch, conditions, expressions, elseExpr); + } + + @Override + protected MySQLExpression and(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); + } + + @Override + protected MySQLExpression orExpr(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); + } + + @Override + protected MySQLExpression not(MySQLExpression expr) { + return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); + } + + @Override + protected MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); + } + + @Override + protected MySQLExpression isNotNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); + } + + @Override + protected MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { + return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); + } + + @Override + protected MySQLExpression generateBooleanExpression() { + return gen.generateBooleanExpression(); + } + + @Override + protected MySQLExpression generateExpressionOfType(CastType type) { + // The MySQL expression generator is untyped, so the type of an arbitrary random expression is pinned by + // wrapping it in a CAST to the requested type. + return new MySQLCastOperation(gen.generateExpression(), type); + } + + @Override + protected CastType inferType(MySQLExpression expr) { + if (expr instanceof MySQLBinaryLogicalOperation || expr instanceof MySQLBinaryComparisonOperation + || expr instanceof MySQLUnaryPostfixOperation || expr instanceof MySQLBetweenOperation + || expr instanceof MySQLInOperation || expr instanceof MySQLExists) { + // Predicates evaluate to the boolean values 0/1, which are signed BIGINT. + return CastType.SIGNED; + } else if (expr instanceof MySQLBinaryOperation) { + // The bit operators &, | and ^ return BIGINT UNSIGNED. + return CastType.UNSIGNED; + } else if (expr instanceof MySQLCastOperation) { + return ((MySQLCastOperation) expr).getType(); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + return inferUnaryPrefixType((MySQLUnaryPrefixOperation) expr); + } else if (expr instanceof MySQLConstant) { + return inferConstantType((MySQLConstant) expr); + } else if (expr instanceof MySQLColumnReference) { + return inferColumnType((MySQLColumnReference) expr); + } else if (expr instanceof MySQLComputableFunction) { + return inferFunctionType((MySQLComputableFunction) expr); + } else if (expr instanceof MySQLCaseOperator) { + return inferCaseType((MySQLCaseOperator) expr); + } + return null; + } + + private CastType inferUnaryPrefixType(MySQLUnaryPrefixOperation op) { + if (op.getOp() == MySQLUnaryPrefixOperator.NOT) { + return CastType.SIGNED; + } + CastType operandType = inferType(op.getExpression()); + if (op.getOp() == MySQLUnaryPrefixOperator.PLUS) { + return operandType; + } + if (op.getOp() == MySQLUnaryPrefixOperator.MINUS) { + if (operandType == CastType.UNSIGNED) { + return CastType.SIGNED; + } else if (operandType == CastType.FLOAT) { + return CastType.DOUBLE; + } else if (operandType != CastType.CHAR) { + return operandType; + } + } + return null; + } + + private CastType inferConstantType(MySQLConstant constant) { + if (constant instanceof MySQLConstant.MySQLIntConstant) { + return constant.isSigned() ? CastType.SIGNED : CastType.UNSIGNED; + } else if (constant instanceof MySQLConstant.MySQLTextConstant) { + return CastType.CHAR; + } else if (constant instanceof MySQLConstant.MySQLDoubleConstant) { + return CastType.DOUBLE; + } + return null; // the NULL constant has no type of its own + } + + private CastType inferColumnType(MySQLColumnReference ref) { + MySQLColumn column = ref.getColumn(); + switch (column.getType()) { + case INT: + return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns + case VARCHAR: + return CastType.CHAR; + case FLOAT: + // FLOAT/DOUBLE columns are created without (M, D) while EET is active (the (M, D) form is deprecated and + // not a valid CAST target), so the plain CAST target matches the column's type. + return CastType.FLOAT; + case DOUBLE: + return CastType.DOUBLE; + case DECIMAL: + // DECIMAL columns may carry (M, D); CAST(... AS DECIMAL(M, D)) reproduces the column's exact type. The + // schema reports (M, D) even for a plain DECIMAL column (defaulting to (10, 0)). + return CastType.decimal(column.getPrecision(), column.getScale()); + default: + return null; + } + } + + private CastType inferFunctionType(MySQLComputableFunction func) { + MySQLExpression[] args = func.getArguments(); + switch (func.getFunction()) { + case BIT_COUNT: + return CastType.SIGNED; + case IF: + // The result type aggregates the types of the two value arguments (the condition does not contribute). + return commonType(args[1], args[2]); + case COALESCE: + case IFNULL: + case LEAST: + case GREATEST: + return commonType(args); + default: + return null; + } + } + + private CastType inferCaseType(MySQLCaseOperator caseOp) { + List branches = new ArrayList<>(caseOp.getExpressions()); + if (caseOp.getElseExpr() != null) { + branches.add(caseOp.getElseExpr()); + } + return commonType(branches.toArray(new MySQLExpression[0])); + } + + /** + * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same + * inferrable type (a conservative under-approximation of MySQL's aggregation rules). Two DECIMAL subexpressions + * with differing {@code (M, D)} therefore yield {@code null} rather than a guessed aggregate. + * + * @param exprs + * the result-type-determining subexpressions + * + * @return the common inferred type of {@code exprs}, or {@code null} if they do not share one + */ + private CastType commonType(MySQLExpression... exprs) { + CastType common = null; + for (MySQLExpression expr : exprs) { + CastType type = inferType(expr); + // equals (not ==) so two DECIMAL types with matching (M, D) but distinct instances compare as equal. + if (type == null || common != null && !type.equals(common)) { + return null; + } + common = type; + } + return common; + } + + @Override + protected boolean isCaseWhenApplicable(MySQLExpression expr) { + // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET + // paper); aggregates are excluded to avoid placing them in invalid contexts. + return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); + } +} diff --git a/src/sqlancer/mysql/oracle/MySQLFuzzer.java b/src/sqlancer/mysql/oracle/MySQLFuzzer.java new file mode 100644 index 000000000..2e361c0b8 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLFuzzer.java @@ -0,0 +1,30 @@ +package sqlancer.mysql.oracle; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.gen.MySQLRandomQuerySynthesizer; + +public class MySQLFuzzer implements TestOracle { + + private final MySQLGlobalState globalState; + + public MySQLFuzzer(MySQLGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public void check() throws Exception { + String s = MySQLVisitor.asString(MySQLRandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + + ';'; + try { + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); + } catch (Error e) { + + } + } + +} diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 871e13334..c1fe893b6 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -1,28 +1,25 @@ package sqlancer.mysql.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.StateToReproduce.MySQLStateToReproduce; -import sqlancer.TestOracle; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLSchema.MySQLTables; -import sqlancer.mysql.MySQLToStringVisitor; +import sqlancer.mysql.MySQLVisitor; import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLConstant; import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLOrderByTerm; -import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; @@ -31,82 +28,36 @@ import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; import sqlancer.mysql.gen.MySQLExpressionGenerator; -public class MySQLPivotedQuerySynthesisOracle implements TestOracle { +public class MySQLPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { - private final MySQLStateToReproduce state; - private final MySQLSchema s; - private MySQLRowValue rw; private List fetchColumns; private List columns; - private final MySQLGlobalState globalState; public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { - this.globalState = globalState; - this.s = globalState.getSchema(); - this.state = (MySQLStateToReproduce) globalState.getState(); + super(globalState); + MySQLErrors.addExpressionErrors(errors); + errors.add("in 'order clause'"); // e.g., Unknown column '2067708013' in 'order clause' + errors.add("in 'EXISTS subquery'"); // e.g., Unknown column '2067708013' in 'EXISTS subquery' (MySQL 8.4+) } @Override - public void check() throws SQLException { - String queryString = getQueryThatContainsAtLeastOneRow(); - - try { - boolean isContainedIn = isContainedIn(queryString); - if (!isContainedIn) { - throw new AssertionError(queryString); - } - } catch (SQLException e) { - if (!e.getMessage().contains("BIGINT value is out of range")) { - throw e; - } - } - - } - - public String getQueryThatContainsAtLeastOneRow() throws SQLException { - MySQLTables randomFromTables = s.getRandomTableNonEmptyTables(); + public Query getRectifiedQuery() throws SQLException { + MySQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); - state.queryTargetedTablesString = randomFromTables.tableNamesAsString(); - MySQLSelect selectStatement = new MySQLSelect(); selectStatement.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); - state.whereClause = selectStatement; columns = randomFromTables.getColumns(); - // for (MySQLTable t : tables) { - // if (t.getRowid() != null) { - // columns.add(t.getRowid()); - // } - // } - rw = randomFromTables.getRandomRowValue(globalState.getConnection(), state); - - // List joinStatements = new ArrayList<>(); - // for (int i = 1; i < tables.size(); i++) { - // SQLite3Expression joinClause = generateWhereClauseThatContainsRowValue(columns, rw); - // Table table = Randomly.fromList(tables); - // tables.remove(table); - // JoinType options; - // if (tables.size() == 2) { - // // allow outer with arbitrary column order (see error: ON clause references - // // tables to its right) - // options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS, JoinType.OUTER); - // } else { - // options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS); - // } - // Join j = new SQLite3Expression.Join(table, joinClause, options); - // joinStatements.add(j); - // } - // selectStatement.setJoinClauses(joinStatements); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + selectStatement.setFromList(tables.stream().map(t -> new MySQLTableReference(t)).collect(Collectors.toList())); fetchColumns = columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); selectStatement.setFetchColumns(fetchColumns); - state.queryTargetedColumnsString = columns.stream().map(c -> c.getFullQualifiedName()) - .collect(Collectors.joining(", ")); - MySQLExpression whereClause = generateWhereClauseThatContainsRowValue(columns, rw); + MySQLExpression whereClause = generateRectifiedExpression(columns, pivotRow); selectStatement.setWhereClause(whereClause); - state.whereClause = selectStatement; - List groupByClause = generateGroupByClause(columns, rw); + List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); MySQLExpression limitClause = generateLimit(); selectStatement.setLimitClause(limitClause); @@ -114,37 +65,13 @@ public String getQueryThatContainsAtLeastOneRow() throws SQLException { MySQLExpression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } - List modifiers = Randomly.subset("STRAIGHT_JOIN", "SQL_SMALL_RESULT", "SQL_BIG_RESULT", "SQL_NO_CACHE"); // "SQL_BUFFER_RESULT", - // "SQL_CALC_FOUND_ROWS", - // "HIGH_PRIORITY" - // TODO: Incorrect usage/placement of 'SQL_BUFFER_RESULT' + List modifiers = Randomly.subset("STRAIGHT_JOIN", "SQL_SMALL_RESULT", "SQL_BIG_RESULT", "SQL_NO_CACHE"); selectStatement.setModifiers(modifiers); - List orderBy = generateOrderBy(columns); - selectStatement.setOrderByExpressions(orderBy); + List orderBy = new MySQLExpressionGenerator(globalState).setColumns(columns) + .generateOrderBys(); + selectStatement.setOrderByClauses(orderBy); - StringBuilder sb2 = new StringBuilder(); - sb2.append("SELECT * FROM (SELECT 1 FROM "); - sb2.append(randomFromTables.tableNamesAsString()); - sb2.append(" WHERE "); - int i = 0; - for (MySQLColumn c : columns) { - if (i++ != 0) { - sb2.append(" AND "); - } - sb2.append(c.getFullQualifiedName()); - if (rw.getValues().get(c).isNull()) { - sb2.append(" IS NULL"); - } else { - sb2.append(" = "); - sb2.append(rw.getValues().get(c).getTextRepresentation()); - } - } - sb2.append(") as result;"); - state.queryThatSelectsRow = sb2.toString(); - - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); - visitor.visit(selectStatement); - return visitor.get(); + return new SQLQueryAdapter(MySQLVisitor.asString(selectStatement), errors); } private List generateGroupByClause(List columns, MySQLRowValue rw) { @@ -156,15 +83,6 @@ private List generateGroupByClause(List columns, M } } - public List generateOrderBy(List columns) { - List orderBys = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber(); i++) { - orderBys.add(new MySQLOrderByTerm(MySQLColumnReference.create(Randomly.fromList(columns), null), - MySQLOrder.getRandomOrder())); - } - return orderBys; - } - private MySQLConstant generateLimit() { if (Randomly.getBoolean()) { return MySQLConstant.createIntConstant(Integer.MAX_VALUE); @@ -175,57 +93,56 @@ private MySQLConstant generateLimit() { private MySQLExpression generateOffset() { if (Randomly.getBoolean()) { - // OFFSET 0 return MySQLConstant.createIntConstantNotAsBoolean(0); } else { return null; } } - private MySQLExpression generateWhereClauseThatContainsRowValue(List columns, MySQLRowValue rw) { + private MySQLExpression generateRectifiedExpression(List columns, MySQLRowValue rw) { MySQLExpression expression = new MySQLExpressionGenerator(globalState).setRowVal(rw).setColumns(columns) .generateExpression(); MySQLConstant expectedValue = expression.getExpectedValue(); + MySQLExpression result; if (expectedValue.isNull()) { - return new MySQLUnaryPostfixOperation(expression, UnaryPostfixOperator.IS_NULL, false); + result = new MySQLUnaryPostfixOperation(expression, UnaryPostfixOperator.IS_NULL, false); } else if (expectedValue.asBooleanNotNull()) { - return expression; + result = expression; } else { - return new MySQLUnaryPrefixOperation(expression, MySQLUnaryPrefixOperator.NOT); + result = new MySQLUnaryPrefixOperation(expression, MySQLUnaryPrefixOperator.NOT); } + rectifiedPredicates.add(result); + return result; } - private boolean isContainedIn(String queryString) throws SQLException { - Statement createStatement; - createStatement = globalState.getConnection().createStatement(); - + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions - sb.append(queryString); + sb.append(query.getUnterminatedQueryString()); sb.append(") as result WHERE "); int i = 0; for (MySQLColumn c : columns) { if (i++ != 0) { sb.append(" AND "); } - sb.append("result." + c.getTable().getName() + c.getName()); - if (rw.getValues().get(c).isNull()) { + sb.append("result."); + sb.append("ref"); + sb.append(i - 1); + if (pivotRow.getValues().get(c).isNull()) { sb.append(" IS NULL"); } else { sb.append(" = "); - sb.append(rw.getValues().get(c).getTextRepresentation()); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); } } String resultingQueryString = sb.toString(); - state.queryString = resultingQueryString; - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(resultingQueryString); - } - try (ResultSet result = createStatement.executeQuery(resultingQueryString)) { - boolean isContainedIn = result.next(); - createStatement.close(); - return isContainedIn; - } + return new SQLQueryAdapter(resultingQueryString, query.getExpectedErrors()); + } + + @Override + protected String getExpectedValues(MySQLExpression expr) { + return MySQLVisitor.asExpectedValues(expr); } } diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java deleted file mode 100644 index c67018344..000000000 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ /dev/null @@ -1,71 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import sqlancer.TestOracle; -import sqlancer.mysql.MySQLErrors; -import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema; -import sqlancer.mysql.MySQLSchema.MySQLTable; -import sqlancer.mysql.MySQLSchema.MySQLTables; -import sqlancer.mysql.ast.MySQLColumnReference; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLSelect; -import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -public abstract class MySQLQueryPartitioningBase implements TestOracle { - - final MySQLGlobalState state; - final Set errors = new HashSet<>(); - - MySQLSchema s; - MySQLTables targetTables; - MySQLExpressionGenerator gen; - MySQLSelect select; - MySQLExpression predicate; - MySQLExpression negatedPredicate; - MySQLExpression isNullPredicate; - - public MySQLQueryPartitioningBase(MySQLGlobalState state) { - this.state = state; - MySQLErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new MySQLExpressionGenerator(state).setColumns(targetTables.getColumns()); - select = new MySQLSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List tableList = tables.stream().map(t -> new MySQLTableReference(t)) - .collect(Collectors.toList()); - // List joins = MySQLJoin.getJoins(tableList, state); - select.setFromList(tableList); - select.setWhereClause(null); - // select.setJoins(joins); - predicate = generatePredicate(); - negatedPredicate = new MySQLUnaryPrefixOperation(predicate, MySQLUnaryPrefixOperator.NOT); - isNullPredicate = new MySQLUnaryPostfixOperation(predicate, - MySQLUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); - } - - List generateFetchColumns() { - return Arrays.asList(MySQLColumnReference.create(targetTables.getColumns().get(0), null)); - } - - MySQLExpression generatePredicate() { - return gen.generateExpression(); - } - -} diff --git a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java deleted file mode 100644 index 4b578a24d..000000000 --- a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLVisitor; - -public class MySQLTLPWhereOracle extends MySQLQueryPartitioningBase { - - public MySQLTLPWhereOracle(MySQLGlobalState state) { - super(state); - } - - @Override - public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = MySQLVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } - select.setOrderByExpressions(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = MySQLVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = MySQLVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = MySQLVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - -} diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java new file mode 100644 index 000000000..7a5bd8a77 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -0,0 +1,74 @@ +package sqlancer.oceanbase; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import sqlancer.common.query.ExpectedErrors; + +public final class OceanBaseErrors { + + private OceanBaseErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) + errors.add("value is out of range"); + errors.add("is not valid for CHARACTER SET"); + errors.add("The observer or zone is not the master"); + errors.add("Incorrect integer value"); + errors.add("Truncated incorrect DOUBLE value"); + errors.add("Invalid numeric"); + errors.add("Data truncated for argument"); + errors.add("Data truncated for column"); + + return errors; + } + + public static List getExpressionErrorsRegex() { + ArrayList errors = new ArrayList<>(); + errors.add(Pattern.compile("Unknown column '.+' in 'order clause'")); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Duplicate entry"); + errors.add("cannot be null"); + errors.add("doesn't have a default value"); + errors.add("Out of range value"); + errors.add("Incorrect double value"); + errors.add("Incorrect float value"); + errors.add("Incorrect int value"); + errors.add("Incorrect tinyint value"); + errors.add("Data truncation"); + errors.add("Bad Number"); + errors.add("The value specified for generated column"); // TODO: do not insert data into generated columns + errors.add("incorrect utf8 value"); + errors.add("Data truncation: %s value is out of range in '%s'"); + errors.add("Incorrect smallint value"); + errors.add("Incorrect bigint value"); + errors.add("Incorrect decimal value"); + errors.add("error parsing regexp"); + errors.add("The observer or zone is not the master"); + errors.add("Incorrect integer value"); + errors.add("Truncated incorrect DOUBLE value"); + errors.add("Data truncated for argument"); + errors.add("Invalid numeric"); + errors.add("Miss column"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java new file mode 100644 index 000000000..f9e57bd24 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java @@ -0,0 +1,155 @@ +package sqlancer.oceanbase; + +import sqlancer.IgnoreMeException; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; + +public class OceanBaseExpectedValueVisitor implements OceanBaseVisitor { + + private final StringBuilder sb = new StringBuilder(); + private int nrTabs; + + private void print(OceanBaseExpression expr) { + OceanBaseToStringVisitor v = new OceanBaseToStringVisitor(); + v.visit(expr); + for (int i = 0; i < nrTabs; i++) { + sb.append("\t"); + } + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(OceanBaseExpression expr) { + nrTabs++; + try { + OceanBaseVisitor.super.visit(expr); + } catch (IgnoreMeException e) { + + } + nrTabs--; + } + + @Override + public void visit(OceanBaseConstant constant) { + print(constant); + } + + @Override + public void visit(OceanBaseColumnReference column) { + print(column); + } + + @Override + public void visit(OceanBaseUnaryPostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(OceanBaseComputableFunction f) { + print(f); + for (OceanBaseExpression expr : f.getArguments()) { + visit(expr); + } + } + + @Override + public void visit(OceanBaseBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public String get() { + return sb.toString(); + } + + @Override + public void visit(OceanBaseSelect select) { + for (OceanBaseExpression j : select.getJoinList()) { + visit(j); + } + if (select.getWhereClause() != null) { + visit(select.getWhereClause()); + } + } + + @Override + public void visit(OceanBaseBinaryComparisonOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(OceanBaseCastOperation op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(OceanBaseInOperation op) { + print(op); + visit(op.getExpr()); + for (OceanBaseExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(OceanBaseOrderByTerm op) { + } + + @Override + public void visit(OceanBaseExists op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(OceanBaseStringExpression op) { + print(op); + } + + @Override + public void visit(OceanBaseTableReference ref) { + } + + @Override + public void visit(OceanBaseAggregate aggr) { + } + + @Override + public void visit(OceanBaseColumnName aggr) { + } + + @Override + public void visit(OceanBaseText func) { + } + + @Override + public void visit(OceanBaseUnaryPrefixOperation op) { + print(op); + visit(op.getExpr()); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseGlobalState.java b/src/sqlancer/oceanbase/OceanBaseGlobalState.java new file mode 100644 index 000000000..fa483072b --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseGlobalState.java @@ -0,0 +1,19 @@ + +package sqlancer.oceanbase; + +import java.sql.SQLException; + +import sqlancer.SQLGlobalState; + +public class OceanBaseGlobalState extends SQLGlobalState { + + @Override + protected OceanBaseSchema readSchema() throws SQLException { + return OceanBaseSchema.fromConnection(getConnection(), getDatabaseName()); + } + + public boolean usesPQS() { + return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == OceanBaseOracleFactory.PQS); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java new file mode 100644 index 000000000..949e5158b --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -0,0 +1,30 @@ +package sqlancer.oceanbase; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT + + ", default host: " + OceanBaseOptions.DEFAULT_HOST + ")") +public class OceanBaseOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 2881; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(OceanBaseOracleFactory.TLP_WHERE); + + @Parameter(names = { "--query-timeout" }, description = "Query timeout") + public int queryTimeout = 1000000000; + @Parameter(names = { "--transaction-timeout" }, description = "Transaction timeout") + public int trxTimeout = 1000000000; + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java new file mode 100644 index 000000000..b1ab1cb5b --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java @@ -0,0 +1,50 @@ +package sqlancer.oceanbase; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; + +public enum OceanBaseOracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + NoREC { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()) + .with("canceling statement due to statement timeout").with("unmatched parentheses") + .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") + .with("range out of order in character class").with("unrecognized character after ") + .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") + .with("malformed number or name after").with("digit expected after").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBasePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java new file mode 100644 index 000000000..b7c5794f1 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -0,0 +1,167 @@ +package sqlancer.oceanbase; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.oceanbase.gen.OceanBaseAlterTable; +import sqlancer.oceanbase.gen.OceanBaseDeleteGenerator; +import sqlancer.oceanbase.gen.OceanBaseDropIndex; +import sqlancer.oceanbase.gen.OceanBaseInsertGenerator; +import sqlancer.oceanbase.gen.OceanBaseTableGenerator; +import sqlancer.oceanbase.gen.OceanBaseTruncateTableGenerator; +import sqlancer.oceanbase.gen.OceanBaseUpdateGenerator; +import sqlancer.oceanbase.gen.datadef.OceanBaseIndexGenerator; + +@AutoService(DatabaseProvider.class) +public class OceanBaseProvider extends SQLProviderAdapter { + + public OceanBaseProvider() { + super(OceanBaseGlobalState.class, OceanBaseOptions.class); + } + + enum Action implements AbstractAction { + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), INSERT(OceanBaseInsertGenerator::insertRow), + CREATE_INDEX(OceanBaseIndexGenerator::create), ALTER_TABLE(OceanBaseAlterTable::create), + TRUNCATE_TABLE(OceanBaseTruncateTableGenerator::generate), + SELECT_INFO((g) -> new SQLQueryAdapter( + "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() + + "'")), + CREATE_TABLE((g) -> { + String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); + + return OceanBaseTableGenerator.generate(g, tableName); + }), DELETE(OceanBaseDeleteGenerator::delete), UPDATE(OceanBaseUpdateGenerator::update), + DROP_INDEX(OceanBaseDropIndex::generate); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws Exception { + return sqlQueryProvider.getQuery(globalState); + } + } + + private static int mapActions(OceanBaseGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed = 0; + switch (a) { + case DROP_INDEX: + nrPerformed = r.getInteger(0, 2); + break; + case SHOW_TABLES: + nrPerformed = r.getInteger(0, 1); + break; + case CREATE_TABLE: + nrPerformed = r.getInteger(0, 1); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 5); + break; + case ALTER_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case TRUNCATE_TABLE: + nrPerformed = r.getInteger(0, 2); + break; + case SELECT_INFO: + nrPerformed = r.getInteger(0, 10); + break; + case DELETE: + nrPerformed = r.getInteger(0, 10); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + } + + @Override + public void generateDatabase(OceanBaseGlobalState globalState) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = OceanBaseTableGenerator.generate(globalState, tableName); + globalState.executeStatement(createTable); + } + + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + OceanBaseProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exception, SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = OceanBaseOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = OceanBaseOptions.DEFAULT_PORT; + } + if (username.endsWith("sys") || username.equals("root")) { + throw new OceanBaseUserCheckException( + "please don't use sys tenant to test! Firstly create tenant then test"); + } + String databaseName = globalState.getDatabaseName(); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + + try (Statement s = con.createStatement()) { + s.execute("set ob_query_timeout=" + globalState.getDbmsSpecificOptions().queryTimeout); + } + try (Statement s = con.createStatement()) { + s.execute("set ob_trx_timeout=" + globalState.getDbmsSpecificOptions().trxTimeout); + } + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "oceanbase"; + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseSchema.java b/src/sqlancer/oceanbase/OceanBaseSchema.java new file mode 100644 index 000000000..7b5b5954f --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseSchema.java @@ -0,0 +1,289 @@ +package sqlancer.oceanbase; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseConstant; + +public class OceanBaseSchema extends AbstractSchema { + + private static final int NR_SCHEMA_READ_TRIES = 10; + + public enum OceanBaseDataType { + INT, VARCHAR, FLOAT, DOUBLE, DECIMAL; + + public static OceanBaseDataType getRandom(OceanBaseGlobalState globalState) { + if (globalState.usesPQS()) { + return Randomly.fromOptions(OceanBaseDataType.INT, OceanBaseDataType.VARCHAR); + } else { + return Randomly.fromOptions(values()); + } + } + + public boolean isNumeric() { + switch (this) { + case INT: + case DOUBLE: + case FLOAT: + case DECIMAL: + return true; + case VARCHAR: + return false; + default: + throw new AssertionError(this); + } + } + + } + + public static class OceanBaseColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isZeroFill; + private final int precision; + public boolean isPartioned; + + public enum CollateSequence { + NOCASE, RTRIM, BINARY; + + public static CollateSequence random() { + return Randomly.fromOptions(values()); + + } + } + + public OceanBaseColumn(String name, OceanBaseDataType columnType, boolean isPrimaryKey, int precision, + boolean isZeroFill) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.precision = precision; + this.isPartioned = true; + this.isZeroFill = isZeroFill; + } + + public int getPrecision() { + return precision; + } + + @Override + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isZeroFill() { + return isZeroFill; + } + + } + + public static class OceanBaseTables extends AbstractTables { + + public OceanBaseTables(List tables) { + super(tables); + } + + public OceanBaseRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", + columnNamesAsString(c -> c.getType() == OceanBaseDataType.FLOAT || c.isZeroFill() + ? "concat(" + c.getTable().getName() + "." + c.getName() + ",'')" + " AS " + + c.getTable().getName() + c.getName() + : c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + + c.getName()), + tableNamesAsString()); + // cast float and zerofill as varchar + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new IgnoreMeException(); + } + for (int i = 0; i < getColumns().size(); i++) { + OceanBaseColumn column = getColumns().get(i); + Object value; + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + OceanBaseConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + if (column.isZeroFill()) { + constant = OceanBaseConstant.createStringConstant("null"); + } else { + constant = OceanBaseConstant.createNullConstant(); + } + } else { + switch (column.getType()) { + case INT: + // cast zerofill as varchar + if (column.isZeroFill()) { + value = randomRowValues.getString(columnIndex); + constant = OceanBaseConstant.createStringConstant((String) value); + } else { + value = randomRowValues.getLong(columnIndex); + constant = OceanBaseConstant.createIntConstant((long) value); + } + break; + case VARCHAR: + value = randomRowValues.getString(columnIndex); + constant = OceanBaseConstant.createStringConstant((String) value); + break; + default: + throw new AssertionError(column.getType()); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new OceanBaseRowValue(this, values); + } + + } + + } + + private static OceanBaseDataType getColumnType(String typeString) { + switch (typeString) { + case "tinyint": + case "smallint": + case "mediumint": + case "int": + case "bigint": + return OceanBaseDataType.INT; + case "varchar": + case "tinytext": + case "mediumtext": + case "text": + case "longtext": + return OceanBaseDataType.VARCHAR; + case "double": + return OceanBaseDataType.DOUBLE; + case "float": + return OceanBaseDataType.FLOAT; + case "decimal": + return OceanBaseDataType.DECIMAL; + default: + throw new AssertionError(typeString); + } + } + + public static class OceanBaseRowValue + extends AbstractRowValue { + + OceanBaseRowValue(OceanBaseTables tables, Map values) { + super(tables, values); + } + + } + + public static class OceanBaseTable + extends AbstractRelationalTable { + + public OceanBaseTable(String tableName, List columns, List indexes) { + super(tableName, columns, indexes, false); + } + + } + + public static final class OceanBaseIndex extends TableIndex { + + private OceanBaseIndex(String indexName) { + super(indexName); + } + + public static OceanBaseIndex create(String indexName) { + return new OceanBaseIndex(indexName); + } + } + + public static OceanBaseSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + Exception ex = null; + for (int i = 0; i < NR_SCHEMA_READ_TRIES; i++) { + try { + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("select TABLE_NAME from information_schema.TABLES where table_schema = '" + + databaseName + "';")) { + while (rs.next()) { + String tableName = rs.getString("TABLE_NAME"); + List databaseColumns = getTableColumns(con, tableName, databaseName); + List indexes = getIndexes(con, tableName, databaseName); + OceanBaseTable t = new OceanBaseTable(tableName, databaseColumns, indexes); + for (OceanBaseColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + return new OceanBaseSchema(databaseTables); + } catch (SQLIntegrityConstraintViolationException e) { + ex = e; + } + } + throw new AssertionError(ex); + } + + private static List getIndexes(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME='%s';", + databaseName, tableName))) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + if (!indexName.equals("PRIMARY")) { + indexes.add(OceanBaseIndex.create(indexName)); + } + } + } + } + return indexes; + } + + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select * from information_schema.columns where table_schema = '" + + databaseName + "' AND TABLE_NAME='" + tableName + "'")) { + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String dataType = rs.getString("DATA_TYPE"); + int precision = rs.getInt("NUMERIC_PRECISION"); + boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); + boolean isZeroFill = rs.getString("COLUMN_TYPE").contains("zerofill"); + + OceanBaseColumn c = new OceanBaseColumn(columnName, getColumnType(dataType), isPrimaryKey, + precision, isZeroFill); + columns.add(c); + } + } + } + return columns; + } + + public OceanBaseSchema(List databaseTables) { + super(databaseTables); + } + + public OceanBaseTables getRandomTableNonEmptyTables() { + return new OceanBaseTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java new file mode 100644 index 000000000..ba306ff90 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -0,0 +1,308 @@ +package sqlancer.oceanbase; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; + +public class OceanBaseToStringVisitor extends ToStringVisitor implements OceanBaseVisitor { + + int ref; + private final Randomly r = new Randomly(); + + @Override + public void visitSpecific(OceanBaseExpression expr) { + OceanBaseVisitor.super.visit(expr); + } + + @Override + public void visit(OceanBaseSelect s) { + sb.append("SELECT "); + if (s.getHint() != null) { + sb.append("/*+ "); + visit(s.getHint(), 0); + sb.append(" */ "); + } + + switch (s.getFromOptions()) { + case DISTINCT: + sb.append("DISTINCT "); + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + sb.append(s.getModifiers().stream().collect(Collectors.joining(" "))); + if (!s.getModifiers().isEmpty()) { + sb.append(" "); + } + if (s.getFetchColumns() == null) { + sb.append("*"); + } else { + for (int i = 0; i < s.getFetchColumns().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getFetchColumns().get(i)); + } + } + sb.append(" FROM "); + for (int i = 0; i < s.getFromList().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getFromList().get(i)); + } + for (OceanBaseExpression j : s.getJoinList()) { + visit(j); + } + + if (s.getWhereClause() != null) { + OceanBaseExpression whereClause = s.getWhereClause(); + sb.append(" WHERE "); + visit(whereClause); + } + if (s.getGroupByExpressions() != null && !s.getGroupByExpressions().isEmpty()) { + sb.append(" "); + sb.append("GROUP BY "); + List groupBys = s.getGroupByExpressions(); + for (int i = 0; i < groupBys.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(groupBys.get(i)); + } + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + } + if (!s.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + List orderBys = s.getOrderByClauses(); + for (int i = 0; i < orderBys.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getOrderByClauses().get(i)); + } + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(OceanBaseConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(OceanBaseColumnReference column) { + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) { + sb.append("concat("); + } + sb.append(column.getColumn().getFullQualifiedName()); + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) { + sb.append(",'')"); + } + if (column.getRef()) { + sb.append(" AS "); + sb.append(column.getColumn().getTable().getName()); + sb.append(column.getColumn().getName()); + } + } + + @Override + public void visit(OceanBaseUnaryPostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" IS "); + if (op.isNegated()) { + sb.append("NOT "); + } + switch (op.getOperator()) { + case IS_FALSE: + sb.append("FALSE"); + break; + case IS_NULL: + if (Randomly.getBoolean()) { + sb.append("UNKNOWN"); + } else { + sb.append("NULL"); + } + break; + case IS_TRUE: + sb.append("TRUE"); + break; + default: + throw new AssertionError(op); + } + } + + @Override + public void visit(OceanBaseComputableFunction f) { + sb.append(f.getFunction().getName()); + sb.append("("); + for (int i = 0; i < f.getArguments().length; i++) { + if (i != 0) { + sb.append(", "); + } + visit(f.getArguments()[i]); + } + sb.append(")"); + } + + @Override + public void visit(OceanBaseBinaryLogicalOperation op) { + sb.append("("); + visit(op.getLeft()); + sb.append(")"); + sb.append(" "); + sb.append(op.getTextRepresentation()); + sb.append(" "); + sb.append("("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseBinaryComparisonOperation op) { + sb.append("("); + visit(op.getLeft()); + sb.append(") "); + sb.append(op.getOp().getTextRepresentation()); + sb.append(" ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseCastOperation op) { + sb.append("CAST("); + visit(op.getExpr()); + sb.append(" AS "); + sb.append(op.getType()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN "); + sb.append("("); + for (int i = 0; i < op.getListElements().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(op.getListElements().get(i)); + } + sb.append(")"); + } + + @Override + public void visit(OceanBaseOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder() == OceanBaseOrder.ASC ? "ASC" : "DESC"); + } + + @Override + public void visit(OceanBaseExists op) { + sb.append(" EXISTS ("); + visit(op.getExpr()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseStringExpression op) { + if (op.getStr().contains("SELECT")) { + sb.append(op.getStr()); + } else { + String str = op.getStr(); + if (!str.isEmpty()) { + sb.append(r.getInteger(0, 100000)); + } else { + sb.append(r.getInteger(0, 1000000)); + } + } + } + + public void visit(OceanBaseStringExpression op, int type) { + sb.append(op.getStr()); + } + + @Override + public void visit(OceanBaseTableReference ref) { + sb.append(ref.getTable().getName()); + } + + @Override + public void visit(OceanBaseAggregate aggr) { + sb.append(aggr.getAggr()); + sb.append("("); + visit(aggr.getExpr()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseColumnName c) { + sb.append(c.getColumn().getName()); + } + + @Override + public void visit(OceanBaseText func) { + visit(func.getExpr()); + sb.append(func.getText()); + } + + @Override + public void visit(OceanBaseUnaryPrefixOperation op) { + sb.append("("); + sb.append(op.getOp().getTextRepresentation()); + sb.append(" "); + visit(op.getExpr()); + sb.append(")"); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseUserCheckException.java b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java new file mode 100644 index 000000000..38136fc8a --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java @@ -0,0 +1,10 @@ +package sqlancer.oceanbase; + +public class OceanBaseUserCheckException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public OceanBaseUserCheckException(String s) { + super(s); + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseVisitor.java b/src/sqlancer/oceanbase/OceanBaseVisitor.java new file mode 100644 index 000000000..91f3549f7 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseVisitor.java @@ -0,0 +1,110 @@ +package sqlancer.oceanbase; + +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; + +public interface OceanBaseVisitor { + + void visit(OceanBaseTableReference ref); + + void visit(OceanBaseConstant constant); + + void visit(OceanBaseColumnReference column); + + void visit(OceanBaseUnaryPostfixOperation column); + + void visit(OceanBaseComputableFunction f); + + void visit(OceanBaseBinaryLogicalOperation op); + + void visit(OceanBaseSelect select); + + void visit(OceanBaseBinaryComparisonOperation op); + + void visit(OceanBaseCastOperation op); + + void visit(OceanBaseInOperation op); + + void visit(OceanBaseOrderByTerm op); + + void visit(OceanBaseExists op); + + void visit(OceanBaseStringExpression op); + + void visit(OceanBaseAggregate aggr); + + void visit(OceanBaseColumnName c); + + void visit(OceanBaseText fun); + + void visit(OceanBaseUnaryPrefixOperation op); + + default void visit(OceanBaseExpression expr) { + if (expr instanceof OceanBaseConstant) { + visit((OceanBaseConstant) expr); + } else if (expr instanceof OceanBaseColumnReference) { + visit((OceanBaseColumnReference) expr); + } else if (expr instanceof OceanBaseUnaryPostfixOperation) { + visit((OceanBaseUnaryPostfixOperation) expr); + } else if (expr instanceof OceanBaseComputableFunction) { + visit((OceanBaseComputableFunction) expr); + } else if (expr instanceof OceanBaseBinaryLogicalOperation) { + visit((OceanBaseBinaryLogicalOperation) expr); + } else if (expr instanceof OceanBaseSelect) { + visit((OceanBaseSelect) expr); + } else if (expr instanceof OceanBaseBinaryComparisonOperation) { + visit((OceanBaseBinaryComparisonOperation) expr); + } else if (expr instanceof OceanBaseCastOperation) { + visit((OceanBaseCastOperation) expr); + } else if (expr instanceof OceanBaseInOperation) { + visit((OceanBaseInOperation) expr); + } else if (expr instanceof OceanBaseOrderByTerm) { + visit((OceanBaseOrderByTerm) expr); + } else if (expr instanceof OceanBaseExists) { + visit((OceanBaseExists) expr); + } else if (expr instanceof OceanBaseStringExpression) { + visit((OceanBaseStringExpression) expr); + } else if (expr instanceof OceanBaseTableReference) { + visit((OceanBaseTableReference) expr); + } else if (expr instanceof OceanBaseAggregate) { + visit((OceanBaseAggregate) expr); + } else if (expr instanceof OceanBaseColumnName) { + visit((OceanBaseColumnName) expr); + } else if (expr instanceof OceanBaseText) { + visit((OceanBaseText) expr); + } else if (expr instanceof OceanBaseUnaryPrefixOperation) { + visit((OceanBaseUnaryPrefixOperation) expr); + } else { + throw new AssertionError(expr); + } + } + + static String asString(OceanBaseExpression expr) { + OceanBaseToStringVisitor visitor = new OceanBaseToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(OceanBaseExpression expr) { + OceanBaseExpectedValueVisitor visitor = new OceanBaseExpectedValueVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/oceanbase/README.md b/src/sqlancer/oceanbase/README.md new file mode 100644 index 000000000..ddac91aed --- /dev/null +++ b/src/sqlancer/oceanbase/README.md @@ -0,0 +1,35 @@ +## Install Oceanbase +There are some methods to install OceanBase. +A method to install a local single-node OceanBase cluster: +### Install OBD by using RPM packages (only for CentOS 7 or later) +```shell +sudo yum install -y yum-utils +sudo yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo +sudo yum install -y ob-deploy +source /etc/profile.d/obd.sh +``` +### Start an OceanBase cluster +```shell +git clone https://github.com/oceanbase/obdeploy.git +cd obdeploy +sudo obd cluster deploy c1 -c ./example/mini-local-example.yaml -A +sudo obd cluster start c1 +``` +After you install OBD, you can run these commands as the root user to start a local single-node OceanBase cluster. +Before you run the commands, make sure that these conditions are met: + +- You have logged on as the root user. +- Ports `2881` and `2882` are available. +- Your server has at least 8 GB of memory. +- Your server has at least 2 CPU cores. + +> **NOTE:** If the preceding conditions are not met, see [OceanBase Deployer](https://github.com/oceanbase/obdeploy/blob/master/README.md). + +> **NOTE:** We do not recommend that you use sys tenant to test. So please deploy clutser with optition -A, will create the test tenant during the bootstrap by using all available resources of the cluster. + +### Then create user for test. + +```shell +mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" +``` +Other methods, see [OceanBase Deployer](https://github.com/oceanbase/obdeploy/blob/master/README.md). diff --git a/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java new file mode 100644 index 000000000..13a0306bb --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java @@ -0,0 +1,25 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseAggregate implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final OceanBaseAggregateFunction aggr; + + public OceanBaseAggregate(OceanBaseExpression expr, OceanBaseAggregateFunction aggr) { + this.expr = expr; + this.aggr = aggr; + } + + public enum OceanBaseAggregateFunction { + COUNT + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public OceanBaseAggregateFunction getAggr() { + return aggr; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java new file mode 100644 index 000000000..6edaa9357 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java @@ -0,0 +1,123 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseBinaryComparisonOperation implements OceanBaseExpression { + + public enum BinaryComparisonOperator { + EQUALS("=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + NOT_EQUALS("!=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.getType() == OceanBaseDataType.INT) { + return OceanBaseConstant.createIntConstant(1 - isEquals.getInt()); + } + return isEquals; + } + }, + LESS("<") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan == null) { + return null; + } + if (lessThan.getType() == OceanBaseDataType.INT && lessThan.getInt() == 0) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant equals = leftVal.isEquals(rightVal); + if (equals.getType() == OceanBaseDataType.INT && equals.getInt() == 1) { + return OceanBaseConstant.createFalse(); + } else { + OceanBaseConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + return OceanBaseUnaryPrefixOperator.NOT.applyNotNull(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant equals = leftVal.isEquals(rightVal); + if (equals.getType() == OceanBaseDataType.INT && equals.getInt() == 1) { + return OceanBaseConstant.createTrue(); + } else { + OceanBaseConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + return OceanBaseUnaryPrefixOperator.NOT.applyNotNull(applyLess); + } + } + }; + + private final String textRepresentation; + + public String getTextRepresentation() { + return textRepresentation; + } + + BinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public abstract OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal); + + public static BinaryComparisonOperator getRandom() { + return Randomly.fromOptions(BinaryComparisonOperator.values()); + } + } + + private final OceanBaseExpression left; + private final OceanBaseExpression right; + private final BinaryComparisonOperator op; + + public OceanBaseBinaryComparisonOperation(OceanBaseExpression left, OceanBaseExpression right, + BinaryComparisonOperator op) { + this.left = left; + this.right = right; + this.op = op; + } + + public OceanBaseExpression getLeft() { + return left; + } + + public BinaryComparisonOperator getOp() { + return op; + } + + public OceanBaseExpression getRight() { + return right; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return op.getExpectedValue(left.getExpectedValue(), right.getExpectedValue()); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java new file mode 100644 index 000000000..2f9f69a87 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java @@ -0,0 +1,111 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.Randomly; + +public class OceanBaseBinaryLogicalOperation implements OceanBaseExpression { + + private final OceanBaseExpression left; + private final OceanBaseExpression right; + private final OceanBaseBinaryLogicalOperator op; + private final String textRepresentation; + + public enum OceanBaseBinaryLogicalOperator { + AND("AND", "&&") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (left.isNull() && right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (left.isNull()) { + if (right.asBooleanNotNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } else if (right.isNull()) { + if (left.asBooleanNotNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } else { + return OceanBaseConstant.createBoolean(left.asBooleanNotNull() && right.asBooleanNotNull()); + } + } + }, + OR("OR", "||") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (!left.isNull() && left.asBooleanNotNull()) { + return OceanBaseConstant.createTrue(); + } else if (!right.isNull() && right.asBooleanNotNull()) { + return OceanBaseConstant.createTrue(); + } else if (left.isNull() || right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } + }, + XOR("XOR") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (left.isNull() || right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + boolean xorVal = left.asBooleanNotNull() ^ right.asBooleanNotNull(); + return OceanBaseConstant.createBoolean(xorVal); + } + }; + + private final String[] textRepresentations; + + OceanBaseBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public abstract OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right); + + public static OceanBaseBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public OceanBaseBinaryLogicalOperation(OceanBaseExpression left, OceanBaseExpression right, + OceanBaseBinaryLogicalOperator op) { + this.left = left; + this.right = right; + this.op = op; + this.textRepresentation = op.getTextRepresentation(); + } + + public OceanBaseExpression getLeft() { + return left; + } + + public OceanBaseBinaryLogicalOperator getOp() { + return op; + } + + public OceanBaseExpression getRight() { + return right; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant leftExpected = left.getExpectedValue(); + OceanBaseConstant rightExpected = right.getExpectedValue(); + if (left.getExpectedValue() == null || right.getExpectedValue() == null) { + return null; + } + return op.apply(leftExpected, rightExpected); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java new file mode 100644 index 000000000..3b5ade9a3 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java @@ -0,0 +1,36 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseCastOperation implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final CastType type; + + public enum CastType { + SIGNED, UNSIGNED; + + public static CastType getRandom() { + return SIGNED; + // return Randomly.fromOptions(CastType.values()); + } + + } + + public OceanBaseCastOperation(OceanBaseExpression expr, CastType type) { + this.expr = expr; + this.type = type; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public CastType getType() { + return type; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expr.getExpectedValue().castAs(type); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java new file mode 100644 index 000000000..ae1a2c001 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java @@ -0,0 +1,17 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema; + +public class OceanBaseColumnName implements OceanBaseExpression { + + private final OceanBaseSchema.OceanBaseColumn column; + + public OceanBaseColumnName(OceanBaseSchema.OceanBaseColumn column) { + this.column = column; + } + + public OceanBaseSchema.OceanBaseColumn getColumn() { + return column; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java new file mode 100644 index 000000000..19753f813 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java @@ -0,0 +1,42 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; + +public class OceanBaseColumnReference implements OceanBaseExpression { + + private final OceanBaseColumn column; + private final OceanBaseConstant value; + private boolean isRef; + + public OceanBaseColumnReference(OceanBaseColumn column, OceanBaseConstant value) { + this.column = column; + this.value = value; + } + + public static OceanBaseColumnReference create(OceanBaseColumn column, OceanBaseConstant value) { + return new OceanBaseColumnReference(column, value); + } + + public OceanBaseColumn getColumn() { + return column; + } + + public OceanBaseConstant getValue() { + return value; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return value; + } + + public OceanBaseColumnReference setRef(boolean isRef) { + this.isRef = isRef; + return this; + } + + public boolean getRef() { + return isRef; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java new file mode 100644 index 000000000..0ac995d32 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java @@ -0,0 +1,281 @@ +package sqlancer.oceanbase.ast; + +import java.util.function.BinaryOperator; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseCastOperation.CastType; + +public class OceanBaseComputableFunction implements OceanBaseExpression { + + private final OceanBaseFunction func; + private final OceanBaseExpression[] args; + + public OceanBaseComputableFunction(OceanBaseFunction func, OceanBaseExpression... args) { + this.func = func; + this.args = args.clone(); + } + + public OceanBaseFunction getFunction() { + return func; + } + + public OceanBaseExpression[] getArguments() { + return args.clone(); + } + + public enum OceanBaseFunction { + + BIT_COUNT(1, "BIT_COUNT") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { + OceanBaseConstant arg = evaluatedArgs[0]; + if (arg.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + long val = arg.castAs(CastType.SIGNED).getInt(); + return OceanBaseConstant.createIntConstant(Long.bitCount(val)); + } + } + + }, + COALESCE(2, "COALESCE") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { + OceanBaseConstant result = OceanBaseConstant.createNullConstant(); + for (OceanBaseConstant arg : args) { + if (!arg.isNull()) { + result = arg; + break; + } + } + return castToMostGeneralType(result, origArgs); + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + IF(3, "IF") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { + OceanBaseConstant cond = args[0]; + OceanBaseConstant left = args[1]; + OceanBaseConstant right = args[2]; + OceanBaseConstant result; + if (cond.isNull() || !cond.asBooleanNotNull()) { + result = right; + } else { + result = left; + } + return castToMostGeneralType(result, new OceanBaseExpression[] { origArgs[1], origArgs[2] }); + } + }, + + IFNULL(2, "IFNULL") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { + OceanBaseConstant result; + if (args[0].isNull()) { + result = args[1]; + } else { + result = args[0]; + } // args[0] and args[1] both null, if type is varchar, return null of varchar + return castToMostGeneralType(result, origArgs); + } + + }, + LEAST(2, "LEAST", true) { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { + return aggregate(evaluatedArgs, args, (min, cur) -> cur.isLessThan(min).asBooleanNotNull() ? cur : min); + } + + }, + GREATEST(2, "GREATEST", true) { + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { + return aggregate(evaluatedArgs, args, (max, cur) -> cur.isLessThan(max).asBooleanNotNull() ? max : cur); + } + }; + + private String functionName; + final int nrArgs; + private final boolean variadic; + + private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, + OceanBaseExpression[] typeExpressions, BinaryOperator op) { + boolean containsNull = Stream.of(evaluatedArgs).anyMatch(arg -> arg.isNull()); + if (containsNull) { + // IFNULL(GREATEST('aa',NULL), 0) -> '0' + // case1:c1 is float，value is NULL;select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' + // select COALESCE(GREATEST(1, concat(t1.c1)), 1) from t1;->1 + // select COALESCE(GREATEST('0', 1, concat(t1.c1)), 1) from t1;->1 + // select COALESCE(GREATEST('0', concat(t1.c1)), 1) from t1;->'1' + // select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' + // case2: c0 is decimal,value is NULL + // select IFNULL(GREATEST("iffI|2&nBJLQQ", c0, '0'), 1) from t0;->1 + // select IFNULL(GREATEST("iffI|2&nBJLQQ", NULL, '0'), 1) from t0;->'1' + OceanBaseDataType type; + boolean allVarchar = true; + for (OceanBaseExpression expr : typeExpressions) { + if (expr instanceof OceanBaseColumnReference) { + type = ((OceanBaseColumnReference) expr).getColumn().getType(); + if (type == OceanBaseDataType.FLOAT) { + type = OceanBaseDataType.VARCHAR; + } + } else { + type = expr.getExpectedValue().getType(); + } + if (type != null && type.isNumeric()) { + allVarchar = false; + break; + } + } + if (allVarchar) { + return OceanBaseConstant.createStringConstant("null"); + } else { + return OceanBaseConstant.createNullConstant(); + } + } + OceanBaseConstant least = evaluatedArgs[1]; + /* + * select least(1,'H8*GPLuBjDj#Xem]W'); -> 0 select least('1','H8*GPLuBjDj#Xem]W'); ->1 select + * LEAST('000000000001', 'b', 1);->0 + */ + OceanBaseDataType dataType = evaluatedArgs[0].getType(); + boolean sameDataType = true; + for (OceanBaseConstant arg : evaluatedArgs) { + if (arg.getType() != dataType) { + sameDataType = false; + break; + } + } + for (OceanBaseConstant arg : evaluatedArgs) { + OceanBaseConstant left; + OceanBaseConstant right; + if (sameDataType) { + left = least; + right = arg; + } else { + // select GREATEST('1.47529e18', -1188315266);->1.47529e18 + if (least.getType() == OceanBaseDataType.VARCHAR) { + left = least.castAsDouble(); + } else { + left = least; + } + if (arg.getType() == OceanBaseDataType.VARCHAR) { + right = arg.castAsDouble(); + } else { + right = arg; + } + } + least = op.apply(right, left); + } + return least; + } + + OceanBaseFunction(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + OceanBaseFunction(int nrArgs, String functionName, boolean variadic) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = variadic; + } + + public int getNrArgs() { + return nrArgs; + } + + public abstract OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args); + + public static OceanBaseFunction getRandomFunction() { + return Randomly.fromOptions(values()); + } + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant[] constants = new OceanBaseConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + } + return func.apply(constants, args); + } + + public static OceanBaseConstant castToMostGeneralType(OceanBaseConstant cons, + OceanBaseExpression... typeExpressions) { + OceanBaseDataType type = getMostGeneralType(typeExpressions); + if (cons.isNull()) { + if (type == OceanBaseDataType.FLOAT || type == OceanBaseDataType.VARCHAR) { + return OceanBaseConstant.createStringConstant("null"); + } else { + return cons; + } + } else { + switch (type) { + case INT: + if (cons.isInt()) { + return cons; + } else { + return OceanBaseConstant.createIntConstant(cons.castAs(CastType.SIGNED).getInt()); + } + case VARCHAR: + return OceanBaseConstant.createStringConstant(cons.castAsString()); + default: + return cons; + } + } + } + + public static OceanBaseDataType getMostGeneralType(OceanBaseExpression... expressions) { + OceanBaseDataType type = null; + for (OceanBaseExpression expr : expressions) { + OceanBaseDataType exprType; + if (expr instanceof OceanBaseColumnReference) { + exprType = ((OceanBaseColumnReference) expr).getColumn().getType(); + if (((OceanBaseColumnReference) expr).getColumn().isZeroFill()) { + exprType = OceanBaseDataType.VARCHAR; + } + } else { + exprType = expr.getExpectedValue().getType(); + } + if (type == null) { + type = exprType; + if (exprType == OceanBaseDataType.FLOAT) { + type = OceanBaseDataType.VARCHAR; + } + } else if (exprType == OceanBaseDataType.VARCHAR || exprType == OceanBaseDataType.FLOAT) { + type = OceanBaseDataType.VARCHAR; + } + + } + return type; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java new file mode 100644 index 000000000..84dadb9e4 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -0,0 +1,616 @@ +package sqlancer.oceanbase.ast; + +import java.math.BigDecimal; +import java.math.BigInteger; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseCastOperation.CastType; + +public abstract class OceanBaseConstant implements OceanBaseExpression { + + public boolean isInt() { + return false; + } + + public boolean isNull() { + return false; + } + + public boolean isDouble() { + return false; + } + + public boolean isEmpty() { + return false; + } + + public abstract static class OceanBaseNoPQSConstant extends OceanBaseConstant { + + @Override + public boolean asBooleanNotNull() { + throw throwException(); + } + + private RuntimeException throwException() { + throw new UnsupportedOperationException("not applicable for PQS evaluation!"); + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + return null; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + throw throwException(); + } + + @Override + public String castAsString() { + throw throwException(); + + } + + @Override + public OceanBaseConstant castAsDouble() { + throw throwException(); + } + + @Override + public OceanBaseDataType getType() { + throw throwException(); + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + throw throwException(); + } + + } + + public static class OceanBaseDoubleConstant extends OceanBaseNoPQSConstant { + + private final double val; + + public OceanBaseDoubleConstant(double val) { + this.val = val; + if (Double.isInfinite(val) || Double.isNaN(val)) { + throw new IgnoreMeException(); + } + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public double getDouble() { + return this.val; + } + + @Override + public long getInt() { + return (long) val; + } + + @Override + public boolean asBooleanNotNull() { + return Double.compare(Math.abs(val), 0.0) != 0; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if (type == CastType.SIGNED) { + long value = (long) val; + if (val - value >= 0.5) { + value = value + 1; + } + return new OceanBaseIntConstant(value, true); + } else if (type == CastType.UNSIGNED) { + long value = (long) val; + if (val - value >= 0.5) { + value = value + 1; + } + return new OceanBaseIntConstant(value, false); + } else { + throw new AssertionError(); + } + } + + @Override + public String castAsString() { + return String.valueOf(new BigDecimal(val)); // select IFNULL(1.713591018E9, '11') -> 1713591018 + } + + @Override + public boolean isDouble() { + return true; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal instanceof OceanBaseIntConstant) { + return OceanBaseConstant.createBoolean(val < rightVal.getInt()); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return OceanBaseConstant.createBoolean(val < rightVal.getDouble()); + } else if (rightVal instanceof OceanBaseTextConstant) { + return isLessThan(rightVal.castAsDouble()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal instanceof OceanBaseIntConstant) { + return OceanBaseConstant.createBoolean(val == rightVal.getInt()); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return OceanBaseConstant.createBoolean(val == rightVal.getDouble()); + } else if (rightVal instanceof OceanBaseTextConstant) { + return isEquals(rightVal.castAsDouble()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.DOUBLE; + } + + } + + public static class OceanBaseTextConstant extends OceanBaseConstant { + + private final String value; + private final boolean singleQuotes; + + public OceanBaseTextConstant(String value) { + this.value = value; + singleQuotes = Randomly.getBoolean(); + + } + + private void checkIfSmallFloatingPointText() { + boolean isSmallFloatingPointText = isString() && asBooleanNotNull() + && castAs(CastType.SIGNED).getInt() == 0; + if (isSmallFloatingPointText) { + throw new IgnoreMeException(); + } + } + + @Override + public boolean isNull() { + return value.equalsIgnoreCase("NULL"); + } + + @Override + public boolean isEmpty() { + // "" " " + if (value.isEmpty()) { + return true; + } else { + for (int i = 0; i < value.length(); i++) { + String sub = value.substring(i, i + 1); + if (!sub.equals(" ")) { + return false; + } + } + return true; + } + } + + @Override + public boolean asBooleanNotNull() { + for (int i = value.length(); i >= 1; i--) { + try { + String substring = value.substring(0, i); + Double val = Double.valueOf(substring); + return val != 0 && !Double.isNaN(val); + } catch (NumberFormatException e) { + // ignore + } + } + return false; + } + + @Override + public String getTextRepresentation() { + StringBuilder sb = new StringBuilder(); + String quotes = singleQuotes ? "'" : "\""; + sb.append(quotes); + String text = value.replace(quotes, quotes + quotes).replace("\\", "\\\\"); + sb.append(text); + sb.append(quotes); + return sb.toString(); + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (isNull() || rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isInt()) { + checkIfSmallFloatingPointText(); + if (asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return castAs(CastType.SIGNED).isEquals(rightVal); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return castAsDouble().isEquals(rightVal); + } else if (rightVal.isString()) { + if (isEmpty() && rightVal.isEmpty()) { + return OceanBaseConstant.createBoolean(true); + } else { + return OceanBaseConstant.createBoolean(value.equalsIgnoreCase(rightVal.getString())); + } + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public String getString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if (isNull()) { + return OceanBaseConstant.createNullConstant(); + } + if (type == CastType.SIGNED || type == CastType.UNSIGNED) { + String value = this.value; + while (value.startsWith(" ") || value.startsWith("\t") || value.startsWith("\n")) { + if (value.startsWith("\n")) { + throw new IgnoreMeException(); + } + value = value.substring(1); + } + for (int i = value.length(); i >= 1; i--) { + try { + String substring = value.substring(0, i); + long val = Long.parseLong(substring); + return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED); + } catch (NumberFormatException e) { + } + } + return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED); + } else { + throw new AssertionError(); + } + } + + @Override + public OceanBaseConstant castAsDouble() { + String value = this.value; + while (value.startsWith(" ") || value.startsWith("\t") || value.startsWith("\n")) { + if (value.startsWith("\n")) { + throw new IgnoreMeException(); + } + value = value.substring(1); + } + for (int i = value.length(); i >= 1; i--) { + try { + String substring = value.substring(0, i); + double val = Double.parseDouble(substring); + return OceanBaseConstant.createDoubleConstant(val); + } catch (NumberFormatException e) { + } + } + return OceanBaseConstant.createIntConstant(0); + } + + @Override + public String castAsString() { + return value; + } + + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.VARCHAR; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (isNull() || rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isInt()) { + if (asBooleanNotNull()) { + throw new IgnoreMeException(); + } + checkIfSmallFloatingPointText(); + return castAs(rightVal.isSigned() ? CastType.SIGNED : CastType.UNSIGNED).isLessThan(rightVal); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return castAsDouble().isLessThan(rightVal); + } else if (rightVal.isString()) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(rightVal); + } + } + } + + public static class OceanBaseIntConstant extends OceanBaseConstant { + + private final long value; + private final String stringRepresentation; + private final boolean isSigned; + + public OceanBaseIntConstant(long value, boolean isSigned) { + this.value = value; + this.isSigned = isSigned; + if (value == 0 && Randomly.getBoolean()) { + stringRepresentation = "FALSE"; + } else if (value == 1 && Randomly.getBoolean()) { + stringRepresentation = "TRUE"; + } else { + if (isSigned) { + stringRepresentation = String.valueOf(value); + } else { + stringRepresentation = Long.toUnsignedString(value); + } + } + } + + public OceanBaseIntConstant(long value, String stringRepresentation) { + this.value = value; + this.stringRepresentation = stringRepresentation; + isSigned = true; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public long getInt() { + return value; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public String getTextRepresentation() { + return stringRepresentation; + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (rightVal.isInt()) { + return OceanBaseConstant.createBoolean(new BigInteger(getStringRepr()) + .compareTo(new BigInteger(((OceanBaseIntConstant) rightVal).getStringRepr())) == 0); + } else if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isString()) { + if (rightVal.asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return isEquals(rightVal.castAs(CastType.SIGNED)); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return OceanBaseConstant.createBoolean(value == rightVal.getDouble()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if (type == CastType.SIGNED) { + return new OceanBaseIntConstant(value, true); + } else if (type == CastType.UNSIGNED) { + return new OceanBaseIntConstant(value, false); + } else { + throw new AssertionError(); + } + } + + @Override + public String castAsString() { + if (isSigned) { + return String.valueOf(value); + } else { + return Long.toUnsignedString(value); + } + } + + @Override + public OceanBaseConstant castAsDouble() { + return this; + } + + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.INT; + } + + @Override + public boolean isSigned() { + return isSigned; + } + + private String getStringRepr() { + if (isSigned) { + return String.valueOf(value); + } else { + return Long.toUnsignedString(value); + } + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (rightVal.isInt()) { + long intVal = rightVal.getInt(); + if (isSigned && rightVal.isSigned()) { + return OceanBaseConstant.createBoolean(value < intVal); + } else { + return OceanBaseConstant.createBoolean(new BigInteger(getStringRepr()) + .compareTo(new BigInteger(((OceanBaseIntConstant) rightVal).getStringRepr())) < 0); + } + } else if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isString()) { + if (rightVal.asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return isLessThan(rightVal.castAs(isSigned ? CastType.SIGNED : CastType.UNSIGNED)); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return OceanBaseConstant.createBoolean(value < rightVal.getDouble()); + } else { + throw new AssertionError(rightVal); + } + } + + } + + public static class OceanBaseNullConstant extends OceanBaseConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public boolean asBooleanNotNull() { + throw new UnsupportedOperationException(this.toString()); + } + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + return OceanBaseConstant.createNullConstant(); + } + + @Override + public OceanBaseConstant castAs(CastType type) { + return this; + } + + @Override + public String castAsString() { + return "NULL"; + } + + @Override + public OceanBaseConstant castAsDouble() { + return this; + } + + @Override + public OceanBaseDataType getType() { + return null; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + return this; + } + + } + + public long getInt() { + throw new UnsupportedOperationException(); + } + + public double getDouble() { + throw new UnsupportedOperationException(); + } + + public boolean isSigned() { + return false; + } + + public String getString() { + throw new UnsupportedOperationException(); + } + + public boolean isString() { + return false; + } + + public static OceanBaseConstant createNullConstant() { + return new OceanBaseNullConstant(); + } + + public static OceanBaseConstant createIntConstant(long value) { + return new OceanBaseIntConstant(value, true); + } + + public static OceanBaseConstant createIntConstant(long value, boolean signed) { + return new OceanBaseIntConstant(value, signed); + } + + public static OceanBaseConstant createUnsignedIntConstant(long value) { + return new OceanBaseIntConstant(value, false); + } + + public static OceanBaseConstant createIntConstantNotAsBoolean(long value) { + return new OceanBaseIntConstant(value, String.valueOf(value)); + } + + public static OceanBaseConstant createDoubleConstant(double value) { + return new OceanBaseDoubleConstant(value); + } + + @Override + public OceanBaseConstant getExpectedValue() { + return this; + } + + public abstract boolean asBooleanNotNull(); + + public abstract String getTextRepresentation(); + + public static OceanBaseConstant createFalse() { + return OceanBaseConstant.createIntConstant(0); + } + + public static OceanBaseConstant createBoolean(boolean isTrue) { + return OceanBaseConstant.createIntConstant(isTrue ? 1 : 0); + } + + public static OceanBaseConstant createTrue() { + return OceanBaseConstant.createIntConstant(1); + } + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract OceanBaseConstant isEquals(OceanBaseConstant rightVal); + + public abstract OceanBaseConstant castAs(CastType type); + + public abstract String castAsString(); + + public abstract OceanBaseConstant castAsDouble(); + + public static OceanBaseConstant createStringConstant(String string) { + return new OceanBaseTextConstant(string); + } + + public abstract OceanBaseDataType getType(); + + protected abstract OceanBaseConstant isLessThan(OceanBaseConstant rightVal); + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseExists.java b/src/sqlancer/oceanbase/ast/OceanBaseExists.java new file mode 100644 index 000000000..18595d261 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseExists.java @@ -0,0 +1,30 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseExists implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final OceanBaseConstant expected; + + public OceanBaseExists(OceanBaseExpression expr, OceanBaseConstant expectedValue) { + this.expr = expr; + this.expected = expectedValue; + } + + public OceanBaseExists(OceanBaseExpression expr) { + this.expr = expr; + this.expected = expr.getExpectedValue(); + if (expected == null) { + throw new AssertionError(); + } + } + + public OceanBaseExpression getExpr() { + return expr; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expected; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseExpression.java b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java new file mode 100644 index 000000000..b721d711f --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java @@ -0,0 +1,12 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; + +public interface OceanBaseExpression extends Expression { + + default OceanBaseConstant getExpectedValue() { + throw new AssertionError("PQS not supported for this operator"); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java new file mode 100644 index 000000000..c4caec195 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java @@ -0,0 +1,56 @@ +package sqlancer.oceanbase.ast; + +import java.util.List; + +public class OceanBaseInOperation implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final List listElements; + private final boolean isTrue; + + public OceanBaseInOperation(OceanBaseExpression expr, List listElements, boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant leftVal = expr.getExpectedValue(); + if (leftVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + boolean isNull = false; + for (OceanBaseExpression rightExpr : listElements) { + OceanBaseConstant rightVal = rightExpr.getExpectedValue(); + + OceanBaseConstant convertedRightVal = rightVal; + OceanBaseConstant isEquals = leftVal.isEquals(convertedRightVal); + if (isEquals.isNull()) { + isNull = true; + } else { + if (isEquals.getInt() == 1) { + return OceanBaseConstant.createBoolean(isTrue); + } + } + } + if (isNull) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createBoolean(!isTrue); + } + + } + + public boolean isTrue() { + return isTrue; + } +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseJoin.java b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java new file mode 100644 index 000000000..4855c0808 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java @@ -0,0 +1,17 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.common.ast.newast.Join; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseJoin implements OceanBaseExpression, Join { + + @Override + public OceanBaseConstant getExpectedValue() { + throw new UnsupportedOperationException(); + } + + @Override + public void setOnClause(OceanBaseExpression onClause) { + } +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java b/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java new file mode 100644 index 000000000..b8f2d029a --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java @@ -0,0 +1,36 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.Randomly; + +public class OceanBaseOrderByTerm implements OceanBaseExpression { + + private final OceanBaseOrder order; + private final OceanBaseExpression expr; + + public enum OceanBaseOrder { + ASC, DESC; + + public static OceanBaseOrder getRandomOrder() { + return Randomly.fromOptions(OceanBaseOrder.values()); + } + } + + public OceanBaseOrderByTerm(OceanBaseExpression expr, OceanBaseOrder order) { + this.expr = expr; + this.order = order; + } + + public OceanBaseOrder getOrder() { + return order; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + @Override + public OceanBaseConstant getExpectedValue() { + throw new AssertionError(this); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java new file mode 100644 index 000000000..c4bbc1b87 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -0,0 +1,82 @@ +package sqlancer.oceanbase.ast; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseSelect extends SelectBase + implements OceanBaseExpression, Select { + + private SelectType fromOptions = SelectType.ALL; + private List modifiers = Collections.emptyList(); + private List groupBys = new ArrayList<>(); + private OceanBaseStringExpression hint; + + public enum SelectType { + DISTINCT, ALL; + } + + public void setSelectType(SelectType fromOptions) { + this.setFromOptions(fromOptions); + } + + public SelectType getFromOptions() { + return fromOptions; + } + + public void setFromOptions(SelectType fromOptions) { + this.fromOptions = fromOptions; + } + + @Override + public void setGroupByClause(List groupBys) { + this.groupBys = groupBys; + } + + @Override + public List getGroupByClause() { + return this.groupBys; + } + + public void setModifiers(List modifiers) { + this.modifiers = modifiers; + } + + public List getModifiers() { + return modifiers; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return null; + } + + public void setHint(OceanBaseStringExpression hint) { + this.hint = hint; + } + + public OceanBaseStringExpression getHint() { + return hint; + } + + @Override + public void setJoinClauses(List joinStatements) { + } + + @Override + public List getJoinClauses() { + return List.of(); + } + + @Override + public String asString() { + return OceanBaseVisitor.asString(this); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java b/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java new file mode 100644 index 000000000..3c1a55176 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java @@ -0,0 +1,22 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseStringExpression implements OceanBaseExpression { + + private final String str; + private final OceanBaseConstant expectedValue; + + public OceanBaseStringExpression(String str, OceanBaseConstant expectedValue) { + this.str = str; + this.expectedValue = expectedValue; + } + + public String getStr() { + return str; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expectedValue; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java b/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java new file mode 100644 index 000000000..8102ca1cc --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java @@ -0,0 +1,17 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseTableReference implements OceanBaseExpression { + + private final OceanBaseTable table; + + public OceanBaseTableReference(OceanBaseTable table) { + this.table = table; + } + + public OceanBaseTable getTable() { + return table; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseText.java b/src/sqlancer/oceanbase/ast/OceanBaseText.java new file mode 100644 index 000000000..f64cba17c --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseText.java @@ -0,0 +1,26 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseText implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final String text; + private final boolean prefix; + + public OceanBaseText(OceanBaseExpression expr, String text, boolean prefix) { + this.expr = expr; + this.text = text; + this.prefix = prefix; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + public boolean isPrefix() { + return prefix; + } +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java new file mode 100644 index 000000000..a2c3b6a57 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java @@ -0,0 +1,58 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseUnaryPostfixOperation implements OceanBaseExpression { + + private final OceanBaseExpression expression; + private final UnaryPostfixOperator operator; + private boolean negate; + + public enum UnaryPostfixOperator { + IS_NULL, IS_TRUE, IS_FALSE; + } + + public OceanBaseUnaryPostfixOperation(OceanBaseExpression expr, UnaryPostfixOperator op, boolean negate) { + this.expression = expr; + this.operator = op; + this.setNegate(negate); + } + + public OceanBaseExpression getExpression() { + return expression; + } + + public UnaryPostfixOperator getOperator() { + return operator; + } + + public boolean isNegated() { + return negate; + } + + public void setNegate(boolean negate) { + this.negate = negate; + } + + @Override + public OceanBaseConstant getExpectedValue() { + boolean val; + OceanBaseConstant expectedValue = expression.getExpectedValue(); + switch (operator) { + case IS_NULL: + val = expectedValue.isNull(); + break; + case IS_FALSE: + val = !expectedValue.isNull() && !expectedValue.asBooleanNotNull(); + break; + case IS_TRUE: + val = !expectedValue.isNull() && expectedValue.asBooleanNotNull(); + break; + default: + throw new AssertionError(operator); + } + if (negate) { + val = !val; + } + return OceanBaseConstant.createIntConstant(val ? 1 : 0); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java new file mode 100644 index 000000000..e659f75be --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java @@ -0,0 +1,94 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseUnaryPrefixOperation extends UnaryOperatorNode + implements OceanBaseExpression { + + public enum OceanBaseUnaryPrefixOperator implements Operator { + NOT("!", "NOT") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + return OceanBaseConstant.createIntConstant(expr.asBooleanNotNull() ? 0 : 1); + } + }, + PLUS("+") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + return expr; + } + }, + MINUS("-") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + if (expr.isString()) { + throw new IgnoreMeException(); + } else if (expr.isInt()) { + if (!expr.isSigned()) { + throw new IgnoreMeException(); + } + return OceanBaseConstant.createIntConstant(-expr.getInt()); + } else if (expr.isDouble()) { + return OceanBaseConstant.createDoubleConstant(-expr.getDouble()); + } else { + throw new AssertionError(expr); + } + } + }; + + private String[] textRepresentations; + + OceanBaseUnaryPrefixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public abstract OceanBaseConstant applyNotNull(OceanBaseConstant expr); + + public static OceanBaseUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + } + + public OceanBaseUnaryPrefixOperation(OceanBaseExpression expr, OceanBaseUnaryPrefixOperator op) { + super(expr, op); + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant subExprVal = expr.getExpectedValue(); + if (op == OceanBaseUnaryPrefixOperator.PLUS) { + if (subExprVal.isNull() && subExprVal.getType() == null) { + return OceanBaseConstant.createNullConstant(); + } else { + return subExprVal; + } + } + if (subExprVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return op.applyNotNull(subExprVal); + } + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.PREFIX; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public OceanBaseUnaryPrefixOperator getOp() { + return op; + } +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java new file mode 100644 index 000000000..7e1a69dfc --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java @@ -0,0 +1,74 @@ +package sqlancer.oceanbase.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseAlterTable { + + private final OceanBaseSchema schema; + private final StringBuilder sb = new StringBuilder(); + boolean couldAffectSchema; + private List selectedActions; + + public OceanBaseAlterTable(OceanBaseSchema newSchema) { + this.schema = newSchema; + } + + public static SQLQueryAdapter create(OceanBaseGlobalState globalState) { + return new OceanBaseAlterTable(globalState.getSchema()).create(); + } + + private enum Action { + COMPRESSION; + + private String[] potentialErrors; + + Action(String... couldCauseErrors) { + this.potentialErrors = couldCauseErrors.clone(); + } + + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = ExpectedErrors.from("does not support the create option", "doesn't have this option", + "is not supported for this operation", "Data truncation", "Specified key was too long"); + errors.add("Data truncated for functional index "); + sb.append("ALTER TABLE "); + OceanBaseTable table = schema.getRandomTable(); + sb.append(table.getName()); + sb.append(" "); + List list = new ArrayList<>(Arrays.asList(Action.values())); + selectedActions = Randomly.subset(list); + int i = 0; + for (Action a : selectedActions) { + if (i++ != 0) { + sb.append(", "); + } + switch (a) { + case COMPRESSION: + sb.append("COMPRESSION "); + sb.append("'"); + sb.append(Randomly.fromOptions("LZ4_1.0", "NONE")); + sb.append("'"); + break; + default: + break; + } + } + for (Action a : selectedActions) { + for (String error : a.potentialErrors) { + errors.add(error); + } + } + return new SQLQueryAdapter(sb.toString(), errors, couldAffectSchema); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java new file mode 100644 index 000000000..ec1aa7fc0 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -0,0 +1,48 @@ +package sqlancer.oceanbase.gen; + +import java.util.Arrays; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseDeleteGenerator extends AbstractDeleteGenerator { + + private final OceanBaseGlobalState globalState; + private final Randomly r; + + public OceanBaseDeleteGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter delete(OceanBaseGlobalState globalState) { + return new OceanBaseDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + OceanBaseTable randomTable = globalState.getSchema().getRandomTable(); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState) + .setColumns(randomTable.getColumns()); + sb.append("DELETE"); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); + } + sb.append(" FROM "); + sb.append(randomTable.getName()); + if (Randomly.getBoolean()) { + appendWhereClause(OceanBaseVisitor.asString(gen.generateExpression())); + OceanBaseErrors.addExpressionErrors(errors); + } + errors.addAll(Arrays.asList("doesn't have this option", "Truncated incorrect DOUBLE value", + "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", + "Data truncated for functional index", "Incorrect value", "Out of range value for column", + "Data truncation:")); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java new file mode 100644 index 000000000..58a17d3ac --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java @@ -0,0 +1,28 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public final class OceanBaseDropIndex { + + private OceanBaseDropIndex() { + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState) { + OceanBaseTable table = globalState.getSchema().getRandomTable(); + if (!table.hasIndexes()) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + sb.append(table.getRandomIndex().getIndexName()); + sb.append(" ON "); + sb.append(table.getName()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("LOCK=NONE is not supported", + "ALGORITHM=INPLACE is not supported", "Data truncation", "Data truncated for functional index")); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java new file mode 100644 index 000000000..42b144c93 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -0,0 +1,368 @@ +package sqlancer.oceanbase.gen; + +import java.sql.Connection; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseRowValue; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseAggregate.OceanBaseAggregateFunction; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation.BinaryComparisonOperator; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseConstant.OceanBaseDoubleConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseJoin; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator + implements NoRECGenerator, + TLPWhereGenerator { + + private OceanBaseGlobalState state; + private OceanBaseRowValue rowVal; + private List tables; + + public OceanBaseExpressionGenerator(OceanBaseGlobalState state) { + this.state = state; + } + + public OceanBaseExpressionGenerator setCon(Connection con) { + return this; + } + + public OceanBaseExpressionGenerator setState(OceanBaseGlobalState state) { + this.state = state; + return this; + } + + public OceanBaseExpressionGenerator setOceanBaseColumns(List columns) { + return this; + } + + public OceanBaseExpressionGenerator setRowVal(OceanBaseRowValue rowVal) { + this.rowVal = rowVal; + return this; + } + + private enum Actions { + COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, + BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, EXISTS; + } + + @Override + public OceanBaseExpression generateExpression(int depth) { + if (depth >= state.getOptions().getMaxExpressionDepth()) { + return generateLeafNode(); + } + switch (Randomly.fromOptions(Actions.values())) { + case COLUMN: + return generateColumn(); + case LITERAL: + return generateConstant(); + case UNARY_PREFIX_OPERATION: + OceanBaseExpression subExpr = generateExpression(depth + 1); + OceanBaseUnaryPrefixOperator random = OceanBaseUnaryPrefixOperator.getRandom(); + return new OceanBaseUnaryPrefixOperation(subExpr, random); + case UNARY_POSTFIX: + return new OceanBaseUnaryPostfixOperation(generateExpression(depth + 1), + Randomly.fromOptions(OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.values()), + Randomly.getBoolean()); + case COMPUTABLE_FUNCTION: + return getComputableFunction(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return new OceanBaseBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), + OceanBaseBinaryLogicalOperator.getRandom()); + case BINARY_COMPARISON_OPERATION: + return new OceanBaseBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), + BinaryComparisonOperator.getRandom()); + case CAST: + return new OceanBaseCastOperation(generateExpression(depth + 1), + OceanBaseCastOperation.CastType.getRandom()); + case IN_OPERATION: + OceanBaseExpression expr = generateExpression(depth + 1); + List rightList = new ArrayList<>(); + for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { + rightList.add(generateExpression(depth + 1)); + } + return new OceanBaseInOperation(expr, rightList, Randomly.getBoolean()); + case EXISTS: + return getExists(); + default: + throw new AssertionError(); + } + } + + private OceanBaseExpression getExists() { + if (Randomly.getBoolean()) { + return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1", OceanBaseConstant.createTrue())); + } else { + return new OceanBaseExists( + new OceanBaseStringExpression("SELECT 1 from dual wHERE FALSE", OceanBaseConstant.createFalse())); + } + } + + private OceanBaseExpression getComputableFunction(int depth) { + OceanBaseFunction func = OceanBaseFunction.getRandomFunction(); + int nrArgs = func.getNrArgs(); + if (func.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + OceanBaseExpression[] args = new OceanBaseExpression[nrArgs]; + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1); + } + return new OceanBaseComputableFunction(func, args); + } + + private enum ConstantType { + INT, NULL, STRING, DOUBLE; + + public static ConstantType[] valuesPQS() { + return new ConstantType[] { INT, NULL, STRING }; + } + } + + @Override + public OceanBaseExpression generateConstant() { + ConstantType[] values; + if (state.usesPQS()) { + values = ConstantType.valuesPQS(); + } else { + values = ConstantType.values(); + } + OceanBaseConstant constant; + switch (Randomly.fromOptions(values)) { + case INT: + return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); + case NULL: + return OceanBaseConstant.createNullConstant(); + case STRING: + String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); + constant = OceanBaseConstant.createStringConstant(string); + return constant; + case DOUBLE: + double val = state.getRandomly().getDouble(); + constant = new OceanBaseDoubleConstant(val); + return constant; + default: + throw new AssertionError(); + } + } + + @Override + public OceanBaseExpression generateColumn() { + OceanBaseColumn c = Randomly.fromList(columns); + OceanBaseConstant val; + if (rowVal == null) { + val = OceanBaseConstant.createNullConstant(); + } else { + val = rowVal.getValues().get(c); + } + return OceanBaseColumnReference.create(c, val); + } + + public OceanBaseExpression generateConstant(OceanBaseColumn col) { + OceanBaseConstant constant; + switch (col.getType().name()) { + case "INT": + return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); + case "NULL": + return OceanBaseConstant.createNullConstant(); + case "VARCHAR": + String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); + constant = OceanBaseConstant.createStringConstant(string); + return constant; + case "DOUBLE": + double val = state.getRandomly().getDouble(); + constant = new OceanBaseDoubleConstant(val); + return constant; + case "FLOAT": + val = state.getRandomly().getDouble(); + constant = new OceanBaseDoubleConstant(val); + return constant; + case "DECIMAL": + val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + default: + throw new AssertionError(); + } + } + + @Override + public OceanBaseExpression negatePredicate(OceanBaseExpression predicate) { + return new OceanBaseUnaryPrefixOperation(predicate, OceanBaseUnaryPrefixOperator.NOT); + } + + @Override + public OceanBaseExpression isNull(OceanBaseExpression expr) { + return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, + false); + } + + @Override + public OceanBaseExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public OceanBaseExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public OceanBaseSelect generateSelect() { + return new OceanBaseSelect(); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(OceanBaseSelect select, OceanBaseExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + OceanBaseExpression aggr = new OceanBaseAggregate( + new OceanBaseColumnReference(new OceanBaseColumn("*", OceanBaseDataType.INT, false, 0, false), + null), + OceanBaseAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + List allColumns = columns.stream().map((c) -> new OceanBaseColumnReference(c, null)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(OceanBaseSelect select, OceanBaseExpression whereCondition) { + OceanBaseExpression expr = getTrueExpr(whereCondition); + + OceanBaseText asText = new OceanBaseText(expr, " as count", false); + select.setFetchColumns(List.of(asText)); + select.setSelectType(OceanBaseSelect.SelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as asdf"; + } + + private enum Option { + TRUE, FALSE_NULL, NOT_NOT_TRUE, NOT_FALSE_NOT_NULL, IF, IFNULL, COALESCE + }; + + private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition) { + // we can treat "is true" as combinations of "is flase" and "not","is not true" and "not",etc. + OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE, false); + + OceanBaseUnaryPostfixOperation isFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, false); + + OceanBaseUnaryPostfixOperation isNotFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, true); + + OceanBaseUnaryPostfixOperation isNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + + OceanBaseUnaryPostfixOperation isNotNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, true); + + OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); + Option a = Randomly.fromOptions(Option.values()); + switch (a) { + case TRUE: + expr = isTrue; + break; + case FALSE_NULL: + // not((is false) or (is null)) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseBinaryLogicalOperation(isFalse, isNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_NOT_TRUE: + // not(not(is true))) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseUnaryPrefixOperation(isTrue, + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_FALSE_NOT_NULL: + // (is not false) and (is not null) + expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); + break; + case IF: + // if(1, xx is true, 0) + OceanBaseExpression[] args = new OceanBaseExpression[3]; + args[0] = OceanBaseConstant.createIntConstant(1); + args[1] = isTrue; + args[2] = OceanBaseConstant.createIntConstant(0); + expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); + break; + case IFNULL: + // ifnull(null, xx is true) + OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; + ifArgs[0] = OceanBaseConstant.createNullConstant(); + ifArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); + break; + case COALESCE: + // coalesce(null, xx is true) + OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; + coalesceArgs[0] = OceanBaseConstant.createNullConstant(); + coalesceArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); + break; + default: + expr = isTrue; + break; + } + return expr; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + return columns.stream().map(c -> new OceanBaseColumnReference(c, null)).collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java new file mode 100644 index 000000000..8c5b66841 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java @@ -0,0 +1,125 @@ +package sqlancer.oceanbase.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; + +public class OceanBaseHintGenerator { + private final OceanBaseSelect select; + private final List tables; + private final StringBuilder sb = new StringBuilder(); + private final Randomly r = new Randomly(); + + enum IndexHint { + PDML, NO_PRED_DEDUCE, MERGE_JOIN, HASH_JOIN, NL_JOIN, BNL_JOIN, NO_MERGE_JOIN, NO_HASH_JOIN, NO_NL_JOIN, + NO_BNL_JOIN, HASH_AGG, NL_MATERIALIZATION, LATE_MATERIALIZATION, USE_INDEX, TOPK, LEADING, ORDERED, NO_REWRITE; + } + + public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { + this.select = select; + this.tables = tables; + } + + public static void generateHints(OceanBaseSelect select, List tables) { + new OceanBaseHintGenerator(select, tables).generate(); + + } + + private void generate() { + switch (Randomly.fromOptions(IndexHint.values())) { + case PDML: + sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); + break; + case NO_PRED_DEDUCE: + sb.append("NO_PRED_DEDUCE"); + break; + case MERGE_JOIN: + tablesHint("USE_MERGE "); + break; + case HASH_JOIN: + tablesHint("USE_HASH "); + break; + case NL_JOIN: + tablesHint("USE_NL "); + break; + case BNL_JOIN: + tablesHint("USE_BNL "); + break; + case NO_MERGE_JOIN: + sb.append(" NO_USE_MERGE "); + break; + case NO_HASH_JOIN: + sb.append(" NO_USE_HASH "); + break; + case NO_NL_JOIN: + sb.append(" NO_USE_NL "); + break; + case NO_BNL_JOIN: + sb.append(" NO_USE_BNL "); + break; + case HASH_AGG: + sb.append("USE_HASH_AGGREGATION "); + break; + case NL_MATERIALIZATION: + sb.append("USE_NL_MATERIALIZATION "); + break; + case LATE_MATERIALIZATION: + sb.append("USE_LATE_MATERIALIZATION "); + break; + case USE_INDEX: + indexesHint("INDEX_HINT "); + break; + case TOPK: + sb.append("TOPK (50 50) "); + break; + case LEADING: + tablesHint(" LEADING "); + break; + case ORDERED: + sb.append("ORDERED "); + break; + case NO_REWRITE: + sb.append("NO_REWRITE "); + break; + default: + throw new AssertionError(); + } + + select.setHint(new OceanBaseStringExpression(sb.toString(), + new OceanBaseConstant.OceanBaseTextConstant(sb.toString()))); + } + + private void indexesHint(String string) { + sb.append(string); + sb.append("("); + OceanBaseTable table = Randomly.fromList(tables); + List allIndexes = table.getIndexes(); + if (allIndexes.isEmpty()) { + throw new IgnoreMeException(); + } + List indexSubset = Randomly.nonEmptySubset(allIndexes); + sb.append(table.getName()); + sb.append(", "); + sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); + sb.append(")"); + } + + private void tablesHint(String string) { + sb.append(string); + sb.append("("); + appendTables(); + sb.append(")"); + } + + private void appendTables() { + List tableSubset = Randomly.nonEmptySubset(tables); + sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + } +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java new file mode 100644 index 000000000..79d206f70 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java @@ -0,0 +1,107 @@ +package sqlancer.oceanbase.gen; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseInsertGenerator { + + private final OceanBaseTable table; + private final StringBuilder sb = new StringBuilder(); + private final ExpectedErrors errors = new ExpectedErrors(); + private final OceanBaseGlobalState globalState; + private final Randomly r; + private int type; + + public OceanBaseInsertGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter insertRow(OceanBaseGlobalState globalState) throws SQLException { + if (Randomly.getBoolean()) { + return new OceanBaseInsertGenerator(globalState).generateInsert(); + } else { + return new OceanBaseInsertGenerator(globalState).generateReplace(); + } + } + + private SQLQueryAdapter generateReplace() { + sb.append("REPLACE"); + type = 1; + return generateInto(); + + } + + private SQLQueryAdapter generateInsert() { + sb.append("INSERT"); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); + } + if (Randomly.getBoolean()) { + sb.append(" "); + } + return generateInto(); + } + + private SQLQueryAdapter generateInto() { + sb.append(" INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(") "); + sb.append("VALUES"); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); + int nrRows; + if (Randomly.getBoolean()) { + nrRows = 1; + } else { + nrRows = 1 + Randomly.smallNumber(); + } + for (int row = 0; row < nrRows; row++) { + if (row != 0) { + sb.append(", "); + } + sb.append("("); + for (int c = 0; c < columns.size(); c++) { + if (c != 0) { + sb.append(", "); + } + OceanBaseColumn col = columns.get(c); + sb.append(OceanBaseVisitor.asString(gen.generateConstant(col))); + + } + sb.append(")"); + } + if (Randomly.getBoolean() && type == 0) { + List upcolumns = table.getRandomNonEmptyColumnSubset(); + if (!upcolumns.isEmpty()) { + sb.append(" ON DUPLICATE KEY UPDATE "); + sb.append(upcolumns.get(0).getName()); + sb.append("="); + sb.append(gen.generateConstant(upcolumns.get(0))); + } + } + errors.add("doesn't have a default value"); + errors.add("Data truncation"); + errors.add("Incorrect integer value"); + errors.add("Duplicate entry"); + errors.add("Data truncated for functional index"); + errors.add("Data truncated for column"); + errors.add("cannot be null"); + errors.add("Incorrect decimal value"); + errors.add("Duplicated primary key"); + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java new file mode 100644 index 000000000..2a193c1ae --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java @@ -0,0 +1,275 @@ +package sqlancer.oceanbase.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; + +public class OceanBaseTableGenerator { + + private final StringBuilder sb = new StringBuilder(); + private final boolean allowPrimaryKey; + private boolean setPrimaryKey; + private final String tableName; + private final Randomly r; + private int columnId; + private final List columns = new ArrayList<>(); + private final OceanBaseSchema schema; + private final OceanBaseGlobalState globalState; + + public OceanBaseTableGenerator(OceanBaseGlobalState globalState, String tableName) { + this.tableName = tableName; + this.r = globalState.getRandomly(); + this.schema = globalState.getSchema(); + allowPrimaryKey = Randomly.getBoolean(); + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState, String tableName) { + return new OceanBaseTableGenerator(globalState, tableName).create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE"); + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { + sb.append(" LIKE "); + sb.append(schema.getRandomTable().getName()); + return new SQLQueryAdapter(sb.toString(), true); + } else { + sb.append("("); + for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(); + } + sb.append(")"); + sb.append(" "); + appendTableOptions(); + appendPartitionOptions(); + addCommonErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + } + + private void addCommonErrors(ExpectedErrors list) { + list.add("doesn't have this option"); + list.add("must include all columns"); + list.add("not allowed type for this type of partitioning"); + list.add("doesn't support BLOB/TEXT columns"); + list.add("A BLOB field is not allowed in partition function"); + list.add("Too many keys specified; max 1 keys allowed"); + list.add("The total length of the partitioning fields is too large"); + } + + private enum PartitionOptions { + HASH, KEY + } + + private void appendPartitionOptions() { + sb.append(" PARTITION BY"); + switch (Randomly.fromOptions(PartitionOptions.values())) { + case HASH: + sb.append(" HASH("); + sb.append(Randomly.fromList(columns)); + sb.append(")"); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + break; + case KEY: + sb.append(" KEY"); + sb.append(" ("); + sb.append(Randomly.nonEmptySubset(columns).stream().collect(Collectors.joining(", "))); + sb.append(")"); + break; + default: + throw new AssertionError(); + } + } + + private enum TableOptions { + BS, BLOOM, AUTO_INCREMENT; + + public static List getRandomTableOptions() { + List options; + if (Randomly.getBooleanWithSmallProbability()) { + options = Randomly.subset(TableOptions.values()); + } else { + if (Randomly.getBoolean()) { + options = Collections.emptyList(); + } else { + options = Randomly.nonEmptySubset(Arrays.asList(TableOptions.values()), 0); + } + } + return options; + } + } + + private void appendTableOptions() { + List tableOptions = TableOptions.getRandomTableOptions(); + int i = 0; + for (TableOptions o : tableOptions) { + if (i++ != 0) { + sb.append(", "); + } + switch (o) { + case AUTO_INCREMENT: + sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); + break; + case BLOOM: + sb.append("USE_BLOOM_FILTER = "); + if (Randomly.getBoolean()) { + sb.append(" FALSE "); + } else { + sb.append(" true "); + } + break; + case BS: + sb.append(" BLOCK_SIZE = "); + if (Randomly.getBoolean()) { + sb.append(" 16384 "); + } else { + sb.append(" 32768 "); + } + break; + default: + throw new AssertionError(o); + } + } + } + + private void appendColumn() { + String columnName = DBMSCommon.createColumnName(columnId); + columns.add(columnName); + sb.append(columnName); + appendColumnDefinition(); + columnId++; + } + + private enum ColumnOptions { + NULL_OR_NOT_NULL, UNIQUE, COMMENT, PRIMARY_KEY + } + + private void appendColumnDefinition() { + sb.append(" "); + + OceanBaseDataType randomType = OceanBaseDataType.getRandom(globalState); + boolean isTextType = randomType == OceanBaseDataType.VARCHAR; + appendTypeString(randomType); + sb.append(" "); + boolean isNull = false; + boolean columnHasPrimaryKey = false; + + List columnOptions = Randomly.subset(ColumnOptions.values()); + if (isTextType) { + columnOptions.remove(ColumnOptions.PRIMARY_KEY); + columnOptions.remove(ColumnOptions.UNIQUE); + } + for (ColumnOptions o : columnOptions) { + sb.append(" "); + switch (o) { + case NULL_OR_NOT_NULL: + // PRIMARY KEYs cannot be NULL + if (!columnHasPrimaryKey) { + if (Randomly.getBoolean()) { + sb.append("NULL"); + } + isNull = true; + } else { + sb.append("NOT NULL"); + } + break; + case UNIQUE: + sb.append("UNIQUE"); + if (Randomly.getBoolean()) { + sb.append(" KEY"); + } + break; + case COMMENT: + sb.append(String.format("COMMENT '%s' ", "asdf")); + break; + case PRIMARY_KEY: + // PRIMARY KEYs cannot be NULL + if (allowPrimaryKey && !setPrimaryKey && !isNull) { + sb.append("PRIMARY KEY"); + setPrimaryKey = true; + columnHasPrimaryKey = true; + } + break; + default: + throw new AssertionError(); + } + } + + } + + private void appendTypeString(OceanBaseDataType randomType) { + switch (randomType) { + case DECIMAL: + sb.append("DECIMAL"); + optionallyAddPrecisionAndScale(sb); + break; + case INT: + sb.append(Randomly.fromOptions("TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT")); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(Randomly.getNotCachedInteger(0, 255)); + sb.append(")"); + } + break; + case VARCHAR: + sb.append(Randomly.fromOptions("VARCHAR(500)")); + break; + case FLOAT: + sb.append("FLOAT"); + optionallyAddPrecisionAndScale(sb); + break; + case DOUBLE: + sb.append(Randomly.fromOptions("DOUBLE", "FLOAT")); + optionallyAddPrecisionAndScale(sb); + break; + default: + throw new AssertionError(); + } + if (randomType.isNumeric()) { + if (Randomly.getBoolean() && randomType != OceanBaseDataType.INT) { + sb.append(" UNSIGNED"); + } + if (Randomly.getBoolean()) { + sb.append(" ZEROFILL"); + } + } + } + + public static void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean()) { + sb.append("("); + long m = Randomly.getNotCachedInteger(1, 53); + sb.append(m); + sb.append(", "); + long nCandidate = Randomly.getNotCachedInteger(1, 30); + long n = Math.min(nCandidate, m); + sb.append(n); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java new file mode 100644 index 000000000..04f5d3613 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java @@ -0,0 +1,18 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; + +public final class OceanBaseTruncateTableGenerator { + + private OceanBaseTruncateTableGenerator() { + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState) { + StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); + sb.append(globalState.getSchema().getRandomTable().getName()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("doesn't have this option")); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java new file mode 100644 index 000000000..51816e691 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -0,0 +1,59 @@ +package sqlancer.oceanbase.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseUpdateGenerator extends AbstractUpdateGenerator { + + private final OceanBaseGlobalState globalState; + private OceanBaseExpressionGenerator gen; + private final Randomly r; + + public OceanBaseUpdateGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter update(OceanBaseGlobalState globalState) { + return new OceanBaseUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel(" + r.getInteger(0, 10) + ") enable_parallel_dml*/ "); + } + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + if (Randomly.getBoolean()) { + OceanBaseErrors.addExpressionErrors(errors); + appendWhereClause(OceanBaseVisitor.asString(gen.generateExpression())); + errors.add("Data Too Long"); + } + errors.add("Duplicated primary key"); + OceanBaseErrors.addInsertErrors(errors); + } + + @Override + protected void updateValue(OceanBaseColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant(column)); + } else { + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + OceanBaseErrors.addExpressionErrors(errors); + } + } +} diff --git a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java new file mode 100644 index 000000000..726d9223d --- /dev/null +++ b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java @@ -0,0 +1,134 @@ +package sqlancer.oceanbase.gen.datadef; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseIndexGenerator { + + private final Randomly r; + private StringBuilder sb = new StringBuilder(); + private OceanBaseSchema schema; + private final List columns = new ArrayList<>(); + private final OceanBaseGlobalState globalState; + + public OceanBaseIndexGenerator(OceanBaseSchema schema, Randomly r, OceanBaseGlobalState globalState) { + this.schema = schema; + this.r = r; + this.globalState = globalState; + } + + public static SQLQueryAdapter create(OceanBaseGlobalState globalState) { + return new OceanBaseIndexGenerator(globalState.getSchema(), globalState.getRandomly(), globalState).create(); + } + + public SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + OceanBaseErrors.addExpressionErrors(errors); + sb.append("CREATE "); + sb.append("INDEX "); + sb.append(globalState.getSchema().getFreeIndexName()); + indexType(); + sb.append(" ON "); + OceanBaseTable table = schema.getRandomTable(); + sb.append(table.getName()); + sb.append("("); + List randomColumn = table.getRandomNonEmptyColumnSubset(); + int i = 0; + for (OceanBaseColumn c : randomColumn) { + if (i++ != 0) { + sb.append(", "); + } + c.isPartioned = true; + columns.add(c); + sb.append(c.getName()); + if (Randomly.getBoolean() && c.getType() == OceanBaseDataType.VARCHAR) { + sb.append("("); + sb.append(r.getInteger(1, 5)); + sb.append(")"); + c.isPartioned = false; + } + } + sb.append(")"); + appendPartitionOptions(); + indexOption(); + String string = sb.toString(); + sb = new StringBuilder(); + errors.add("A primary key index cannot be invisible"); + errors.add("Functional index on a column is not supported. Consider using a regular index instead."); + errors.add("Incorrect usage of spatial/fulltext/hash index and explicit index order"); + errors.add("must include all columns"); + errors.add("cannot index the expression"); + errors.add("Data truncation: Truncated incorrect"); + errors.add("a disallowed function."); + errors.add("Data truncation"); + errors.add("Cannot create a functional index on an expression that returns a BLOB or TEXT."); + errors.add("used in key specification without a key length"); + errors.add("can't be used in key specification with the used table type"); + errors.add("Specified key was too long"); + errors.add("out of range"); + errors.add("Data truncated for functional index"); + errors.add("used in key specification without a key length"); + errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration + return new SQLQueryAdapter(string, errors, true); + } + + private void appendPartitionOptions() { + if (Randomly.getBoolean()) { + return; + } + + OceanBaseColumn colIndex = Randomly.fromList(columns); + + if (!colIndex.isPartioned) { + return; + } + + if (colIndex.getType() == OceanBaseDataType.VARCHAR) { + sb.append(" PARTITION BY"); + sb.append(" KEY"); + sb.append(" ("); + String name = colIndex.getName(); + sb.append(name); + sb.append(")"); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + } else if (OceanBaseDataType.INT == colIndex.getType()) { + sb.append(" PARTITION BY"); + sb.append(" HASH("); + String name = colIndex.getName(); + sb.append(name); + sb.append(") "); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + } else { + return; + } + } + + private void indexOption() { + if (Randomly.getBoolean()) { + sb.append(" "); + } + } + + private void indexType() { + if (Randomly.getBoolean()) { + sb.append(" USING "); + sb.append(Randomly.fromOptions("BTREE", "HASH")); + } + } + + public void setNewSchema(OceanBaseSchema schema) { + this.schema = schema; + } +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..8fa0a4bc0 --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java @@ -0,0 +1,163 @@ +package sqlancer.oceanbase.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseRowValue; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; +import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation.UnaryPostfixOperator; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; + +public class OceanBasePivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + private List columns; + + public OceanBasePivotedQuerySynthesisOracle(OceanBaseGlobalState globalState) throws SQLException { + super(globalState); + OceanBaseErrors.addExpressionErrors(errors); + errors.add("in 'order clause'"); + errors.add("value is out of range"); + } + + @Override + public Query getRectifiedQuery() throws SQLException { + OceanBaseTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + List tables = randomFromTables.getTables(); + + OceanBaseSelect selectStatement = new OceanBaseSelect(); + selectStatement.setSelectType(Randomly.fromOptions(OceanBaseSelect.SelectType.values())); + columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + selectStatement + .setFromList(tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList())); + + fetchColumns = columns.stream().map(c -> new OceanBaseColumnReference(c, null)).map(d -> d.setRef(true)) + .collect(Collectors.toList()); + selectStatement.setFetchColumns(fetchColumns); + OceanBaseExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + OceanBaseExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + OceanBaseExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = generateOrderBy(columns); + selectStatement.setOrderByClauses(orderBy); + + return new SQLQueryAdapter(OceanBaseVisitor.asString(selectStatement), errors); + } + + private List generateGroupByClause(List columns, OceanBaseRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> OceanBaseColumnReference.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + public List generateOrderBy(List columns) { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new OceanBaseOrderByTerm(OceanBaseColumnReference.create(Randomly.fromList(columns), null), + OceanBaseOrder.getRandomOrder())); + } + return orderBys; + } + + private OceanBaseConstant generateLimit() { + if (Randomly.getBoolean()) { + return OceanBaseConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private OceanBaseExpression generateOffset() { + if (Randomly.getBoolean()) { + return OceanBaseConstant.createIntConstantNotAsBoolean(0); + } else { + return null; + } + } + + private OceanBaseExpression generateRectifiedExpression(List columns, OceanBaseRowValue rw) { + OceanBaseExpression expression = new OceanBaseExpressionGenerator(globalState).setRowVal(rw).setColumns(columns) + .generateExpression(); + OceanBaseConstant expectedValue = expression.getExpectedValue(); + OceanBaseExpression result; + if (expectedValue.isNull()) { + result = new OceanBaseUnaryPostfixOperation(expression, UnaryPostfixOperator.IS_NULL, false); + } else if (expectedValue.asBooleanNotNull()) { + result = expression; + } else { + result = new OceanBaseUnaryPrefixOperation(expression, OceanBaseUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (OceanBaseColumn c : columns) { + if (i++ != 0) { + sb.append(" AND "); + } + if (pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) { + sb.append("concat("); + } + sb.append("result." + c.getTable().getName() + c.getName()); + if (pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) { + sb.append(",'')"); + } + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, query.getExpectedErrors()); + } + + @Override + protected String getExpectedValues(OceanBaseExpression expr) { + return OceanBaseVisitor.asExpectedValues(expr); + } +} diff --git a/src/sqlancer/postgres/PostgresBugs.java b/src/sqlancer/postgres/PostgresBugs.java new file mode 100644 index 000000000..352d6c1c5 --- /dev/null +++ b/src/sqlancer/postgres/PostgresBugs.java @@ -0,0 +1,10 @@ +package sqlancer.postgres; + +// do not make the fields final to avoid warnings +public final class PostgresBugs { + public static boolean bug18643 = true; + + private PostgresBugs() { + } + +} diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index 55287e273..8985caf0b 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -1,14 +1,18 @@ package sqlancer.postgres; +import sqlancer.IgnoreMeException; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresFunction; import sqlancer.postgres.ast.PostgresInOperation; +import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -16,7 +20,10 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -30,7 +37,8 @@ private void print(PostgresExpression expr) { sb.append("\t"); } sb.append(v.get()); - sb.append(" -- " + expr.getExpectedValue()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); sb.append("\n"); } @@ -40,7 +48,7 @@ private void print(PostgresExpression expr) { // try { // super.visit(expr); // } catch (IgnoreMeException e) { - // + // } // nrTabs--; // } @@ -71,6 +79,15 @@ public void visit(PostgresPrefixOperation op) { visit(op.getExpression()); } + @Override + public void visit(PostgresColumnReference column) { + print(column); + } + + @Override + public void visit(PostgresTableReference tb) { + } + @Override public void visit(PostgresSelect op) { visit(op.getWhereClause()); @@ -81,6 +98,11 @@ public void visit(PostgresOrderByTerm op) { } + @Override + public void visit(PostgresWindowFunction windowFunction) { + throw new IgnoreMeException(); + } + @Override public void visit(PostgresFunction f) { print(f); @@ -154,4 +176,23 @@ public void visit(PostgresFromTable from) { print(from); } + @Override + public void visit(PostgresSubquery subquery) { + print(subquery); + } + + @Override + public void visit(PostgresBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(PostgresLikeOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + } diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 29ee305fe..0f2c7ebdb 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -1,56 +1,60 @@ package sqlancer.postgres; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; -import sqlancer.GlobalState; import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; -public class PostgresGlobalState extends GlobalState { +public class PostgresGlobalState extends SQLGlobalState { - private List operators; - private List collates; - private List opClasses; - private PostgresSchema schema; + public static final char IMMUTABLE = 'i'; + public static final char STABLE = 's'; + public static final char VOLATILE = 'v'; + + private List operators = Collections.emptyList(); + private List collates = Collections.emptyList(); + private List opClasses = Collections.emptyList(); + private List tableAccessMethods = Collections.emptyList(); + // store and allow filtering by function volatility classifications + private final Map functionsAndTypes = new HashMap<>(); + private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); @Override - public void setConnection(Connection con) { + public void setConnection(SQLConnection con) { super.setConnection(con); try { this.opClasses = getOpclasses(getConnection()); this.operators = getOperators(getConnection()); this.collates = getCollnames(getConnection()); + this.tableAccessMethods = getTableAccessMethods(getConnection()); } catch (SQLException e) { throw new AssertionError(e); } } - public void setSchema(PostgresSchema schema) { - this.schema = schema; - } - - public PostgresSchema getSchema() { - return schema; - } - - private List getCollnames(Connection con) throws SQLException { - List opClasses = new ArrayList<>(); + private List getCollnames(SQLConnection con) throws SQLException { + List collNames = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { while (rs.next()) { - opClasses.add(rs.getString(1)); + collNames.add(rs.getString(1)); } } } - return opClasses; + return collNames; } - private List getOpclasses(Connection con) throws SQLException { + private List getOpclasses(SQLConnection con) throws SQLException { List opClasses = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("select opcname FROM pg_opclass;")) { @@ -62,16 +66,31 @@ private List getOpclasses(Connection con) throws SQLException { return opClasses; } - private List getOperators(Connection con) throws SQLException { - List opClasses = new ArrayList<>(); + private List getOperators(SQLConnection con) throws SQLException { + List operators = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { while (rs.next()) { - opClasses.add(rs.getString(1)); + operators.add(rs.getString(1)); } } } - return opClasses; + return operators; + } + + private List getTableAccessMethods(SQLConnection con) throws SQLException { + List tableAccessMethods = new ArrayList<>(); + try (Statement s = con.createStatement()) { + /* + * pg_am includes both index and table access methods so we need to filter with amtype = 't' + */ + try (ResultSet rs = s.executeQuery("SELECT amname FROM pg_am WHERE amtype = 't';")) { + while (rs.next()) { + tableAccessMethods.add(rs.getString(1)); + } + } + } + return tableAccessMethods; } public List getOperators() { @@ -98,4 +117,37 @@ public String getRandomOpclass() { return Randomly.fromList(opClasses); } + public List getTableAccessMethods() { + return tableAccessMethods; + } + + public String getRandomTableAccessMethod() { + return Randomly.fromList(tableAccessMethods); + } + + @Override + public PostgresSchema readSchema() throws SQLException { + return PostgresSchema.fromConnection(getConnection(), getDatabaseName()); + } + + public void addFunctionAndType(String functionName, Character functionType) { + this.functionsAndTypes.put(functionName, functionType); + } + + public Map getFunctionsAndTypes() { + return this.functionsAndTypes; + } + + public void setAllowedFunctionTypes(List types) { + this.allowedFunctionTypes = types; + } + + public void setDefaultAllowedFunctionTypes() { + this.allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + } + + public List getAllowedFunctionTypes() { + return this.allowedFunctionTypes; + } + } diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 0970360ce..ebef13a16 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -1,68 +1,120 @@ package sqlancer.postgres; -import java.sql.SQLException; -import java.util.ArrayList; +import java.io.File; import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; -import sqlancer.MainOptions.DBMSConverter; -import sqlancer.TestOracle; -import sqlancer.postgres.oracle.PostgresNoRECOracle; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; +import sqlancer.DBMSSpecificOptions; -@Parameters -public class PostgresOptions { +@Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT + + ", default host: " + PostgresOptions.DEFAULT_HOST + ")") +public class PostgresOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 5432; + private static Boolean defaultTestTablespaces; - @Parameter(names = "--bulk-insert") + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; - @Parameter(names = "--oracle", converter = DBMSConverter.class) - public List oracle = Arrays.asList(PostgresOracle.QUERY_PARTITIONING); + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for PostgreSQL") + public List oracle = Arrays.asList(PostgresOracleFactory.QUERY_PARTITIONING); - @Parameter(names = "--test-collations", arity = 1) + @Parameter(names = "--connection-timeout", description = "Timeout in seconds for connecting to the server", arity = 1) + public int connectionTimeoutInSeconds; + + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) public boolean testCollations = true; - public enum PostgresOracle { - NOREC { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresNoRECOracle(globalState); - } - }, - PQS { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresPivotedQuerySynthesisOracle(globalState); - } - }, - HAVING { - - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); - oracles.add(new PostgresTLPWhereOracle(globalState)); - oracles.add(new PostgresTLPHavingOracle(globalState)); - oracles.add(new PostgresTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles); - } - }; - - public abstract TestOracle create(PostgresGlobalState globalState) throws SQLException; + @Parameter(names = "--test-tablespaces", description = "Specifies whether to test tablespace creation (default is OS-dependent)", arity = 1) + public boolean testTablespaces; + + @Parameter(names = "--tablespace-path", description = "Base path for tablespace directories (default is OS-dependent)", arity = 1) + public String tablespacePath = getDefaultTablespacePath(); + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the PostgreSQL server", arity = 1) + public String connectionURL = String.format("postgresql://%s:%d/test", PostgresOptions.DEFAULT_HOST, + PostgresOptions.DEFAULT_PORT); + + @Parameter(names = "--extensions", description = "Specifies a comma-separated list of extension names to be created in each test database", arity = 1) + public String extensions = ""; + + private static boolean determineDefaultTablespaceSupport() { + String osName = System.getProperty("os.name").toLowerCase(); + if (osName.contains("linux")) { + System.out.println("[INFO] Linux detected: Enabling tablespace testing by default"); + return true; + } else if (osName.contains("mac") || osName.contains("darwin")) { + System.out.println( + "[INFO] macOS detected: Disabling tablespace testing by default due to different /tmp handling. Override with --test-tablespaces=true and ensure proper directory permissions."); + return false; + } else if (osName.contains("windows")) { + System.out.println( + "[INFO] Windows detected: Disabling tablespace testing by default due to path format differences. Override with --test-tablespaces=true and use --tablespace-path to set a valid Windows path."); + return false; + } else { + System.out.println( + "[INFO] Unknown OS detected: Disabling tablespace testing by default for safety. Override with --test-tablespaces=true if your system supports PostgreSQL tablespaces."); + return false; + } + } + + public static String getDefaultTablespacePath() { + String osName = System.getProperty("os.name").toLowerCase(); + if (osName.contains("windows")) { + // On Windows, use a path in the temp directory + return new File(System.getProperty("java.io.tmpdir"), "postgresql" + File.separator + "tablespace") + .getAbsolutePath(); + } else { + // On Unix-like systems, use /tmp + return "/tmp/postgresql/tablespace"; + } + } + + @Override + public List getTestOracleFactory() { + return oracle; + } + + public String getTablespacePath() { + if (tablespacePath == null || tablespacePath.isBlank()) { + throw new AssertionError("Tablespace path is null or empty. Please configure --tablespace-path"); + } + + File path = new File(tablespacePath); + + // Check if the directory exists or can be created + if (!path.exists() && !path.mkdirs()) { + throw new AssertionError("Cannot create tablespace directory: " + tablespacePath + + ". Please ensure the parent directory exists and you have write permissions."); + } + + // Check if it's actually a directory + if (!path.isDirectory()) { + throw new AssertionError("Tablespace path is not a directory: " + tablespacePath); + } + // Check write permissions + if (!path.canWrite()) { + throw new AssertionError("No write permissions for tablespace directory: " + tablespacePath + + ". Please ensure you have write permissions to this directory."); + } + + return tablespacePath; + } + + public boolean isTestTablespaces() { + // If the user explicitly set the value via command line, use that + // Otherwise, use the OS-dependent default + return testTablespaces || getDefaultTablespaceSupport(); } + private static boolean getDefaultTablespaceSupport() { + if (defaultTestTablespaces == null) { + defaultTestTablespaces = determineDefaultTablespaceSupport(); + } + return defaultTestTablespaces; + } } diff --git a/src/sqlancer/postgres/PostgresOracleFactory.java b/src/sqlancer/postgres/PostgresOracleFactory.java new file mode 100644 index 000000000..7c00021ed --- /dev/null +++ b/src/sqlancer/postgres/PostgresOracleFactory.java @@ -0,0 +1,117 @@ +package sqlancer.postgres; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.PostgresFuzzer; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; + +public enum PostgresOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + WHERE { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + HAVING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(new PostgresTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .with(PostgresCommon.getCommonFetchErrors()).with(PostgresCommon.getCommonInsertUpdateErrors()) + .with(PostgresCommon.getGroupingErrors()).with(PostgresCommon.getCommonInsertUpdateErrors()) + .with(PostgresCommon.getCommonRangeExpressionErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(1).trim(); + if (content.contains("Result") && content.contains("rows=")) { + try { + int ind = content.indexOf("rows="); + long number = Long.parseLong(content.substring(ind + 5).split(" ")[0]); + return Optional.of(number); + } catch (Exception e) { + } + } + return Optional.empty(); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String content = rs.getString(1).trim(); + String[] planPart = content.split("-> "); + String plan = planPart[planPart.length - 1]; + return Optional.of(plan.split(" ")[0].trim()); + }; + return new CERTOracle<>(globalState, gen, errors, rowCountParser, queryPlanParser); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + FUZZER { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + return new PostgresFuzzer(globalState); + } + + }; + +} diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 192aa4cb3..814013df6 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -1,34 +1,35 @@ package sqlancer.postgres; -import java.io.FileWriter; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedList; import java.util.List; +import java.util.Queue; import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; -import sqlancer.GlobalState; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; import sqlancer.MainOptions; -import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.StateToReproduce; -import sqlancer.StateToReproduce.PostgresStateToReproduce; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.gen.PostgresAlterTableGenerator; import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; @@ -36,62 +37,70 @@ import sqlancer.postgres.gen.PostgresDeleteGenerator; import sqlancer.postgres.gen.PostgresDiscardGenerator; import sqlancer.postgres.gen.PostgresDropIndexGenerator; +import sqlancer.postgres.gen.PostgresExplainGenerator; import sqlancer.postgres.gen.PostgresIndexGenerator; import sqlancer.postgres.gen.PostgresInsertGenerator; import sqlancer.postgres.gen.PostgresNotifyGenerator; -import sqlancer.postgres.gen.PostgresQueryCatalogGenerator; import sqlancer.postgres.gen.PostgresReindexGenerator; import sqlancer.postgres.gen.PostgresSequenceGenerator; import sqlancer.postgres.gen.PostgresSetGenerator; import sqlancer.postgres.gen.PostgresStatisticsGenerator; import sqlancer.postgres.gen.PostgresTableGenerator; +import sqlancer.postgres.gen.PostgresTableSpaceGenerator; import sqlancer.postgres.gen.PostgresTransactionGenerator; import sqlancer.postgres.gen.PostgresTruncateGenerator; import sqlancer.postgres.gen.PostgresUpdateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; import sqlancer.postgres.gen.PostgresViewGenerator; -import sqlancer.sqlite3.gen.SQLite3Common; // EXISTS // IN -public final class PostgresProvider extends ProviderAdapter { +@AutoService(DatabaseProvider.class) +public class PostgresProvider extends SQLProviderAdapter { + /** + * Generate only data types and expressions that are understood by PQS. + */ public static boolean generateOnlyKnown; - private PostgresGlobalState globalState; + protected String entryURL; + protected String username; + protected String password; + protected String entryPath; + protected String host; + protected int port; + protected String testURL; + protected String databaseName; + protected String createDatabaseCommand; + protected String extensionsList; public PostgresProvider() { super(PostgresGlobalState.class, PostgresOptions.class); } + protected PostgresProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + public enum Action implements AbstractAction { ANALYZE(PostgresAnalyzeGenerator::create), // ALTER_TABLE(g -> PostgresAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g, generateOnlyKnown)), // CLUSTER(PostgresClusterGenerator::create), // COMMIT(g -> { - Query query; + SQLQueryAdapter query; if (Randomly.getBoolean()) { - query = new QueryAdapter("COMMIT") { - @Override - public boolean couldAffectSchema() { - return true; - } - }; + query = new SQLQueryAdapter("COMMIT", true); } else if (Randomly.getBoolean()) { query = PostgresTransactionGenerator.executeBegin(); } else { - query = new QueryAdapter("ROLLBACK") { - @Override - public boolean couldAffectSchema() { - return true; - } - }; + query = new SQLQueryAdapter("ROLLBACK", true); } return query; }), // CREATE_STATISTICS(PostgresStatisticsGenerator::insert), // DROP_STATISTICS(PostgresStatisticsGenerator::remove), // + ALTER_STATISTICS(PostgresStatisticsGenerator::alter), // DELETE(PostgresDeleteGenerator::create), // DISCARD(PostgresDiscardGenerator::create), // DROP_INDEX(PostgresDropIndexGenerator::create), // @@ -106,34 +115,35 @@ public boolean couldAffectSchema() { StringBuilder sb = new StringBuilder(); sb.append("SET CONSTRAINTS ALL "); sb.append(Randomly.fromOptions("DEFERRED", "IMMEDIATE")); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); }), // - RESET_ROLE((g) -> new QueryAdapter("RESET ROLE")), // + RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // COMMENT_ON(PostgresCommentGenerator::generate), // - RESET((g) -> new QueryAdapter("RESET ALL") /* - * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also - * configuration parameter - */), // + RESET((g) -> new SQLQueryAdapter("RESET ALL") /* + * https://www.postgresql.org/docs/13/sql-reset.html TODO: also + * configuration parameter + */), // NOTIFY(PostgresNotifyGenerator::createNotify), // LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // + EXPLAIN(PostgresExplainGenerator::create), // CREATE_VIEW(PostgresViewGenerator::create), // - QUERY_CATALOG((g) -> PostgresQueryCatalogGenerator.query()); + CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(PostgresGlobalState state) throws SQLException { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(PostgresGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } - private static int mapActions(PostgresGlobalState globalState, Action a) { + protected static int mapActions(PostgresGlobalState globalState, Action a) { Randomly r = globalState.getRandomly(); int nrPerformed; switch (a) { @@ -144,6 +154,9 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { case CREATE_STATISTICS: nrPerformed = r.getInteger(0, 5); break; + case ALTER_STATISTICS: + nrPerformed = r.getInteger(0, 2); + break; case DISCARD: case DROP_INDEX: nrPerformed = r.getInteger(0, 5); @@ -161,7 +174,6 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { case DELETE: case RESET_ROLE: case SET: - case QUERY_CATALOG: nrPerformed = r.getInteger(0, 5); break; case ANALYZE: @@ -181,12 +193,18 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { case CREATE_VIEW: nrPerformed = r.getInteger(0, 2); break; + case CREATE_TABLESPACE: + nrPerformed = globalState.getDbmsSpecificOptions().isTestTablespaces() ? r.getInteger(0, 2) : 0; + break; case UPDATE: nrPerformed = r.getInteger(0, 10); break; case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; + case EXPLAIN: + nrPerformed = r.getInteger(0, 1); + break; default: throw new AssertionError(a); } @@ -195,118 +213,180 @@ private static int mapActions(PostgresGlobalState globalState, Action a) { } @Override - public void generateAndTestDatabase(PostgresGlobalState globalState) throws SQLException { - MainOptions options = globalState.getOptions(); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); - String databaseName = globalState.getDatabaseName(); - Connection con = globalState.getConnection(); - QueryManager manager = globalState.getManager(); - if (options.logEachSelect()) { - logger.writeCurrent(state); - } - globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); - while (globalState.getSchema().getDatabaseTables().size() < 1) { - try { - String tableName = SQLite3Common.createTableName(globalState.getSchema().getDatabaseTables().size()); - Query createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), - generateOnlyKnown, globalState); - if (options.logEachSelect()) { - logger.writeCurrent(createTable.getQueryString()); - } - manager.execute(createTable); - globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); - } catch (IgnoreMeException e) { + public void generateDatabase(PostgresGlobalState globalState) throws Exception { + readFunctions(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + extensionsList = globalState.getDbmsSpecificOptions().extensions; + if (!extensionsList.isEmpty()) { + String[] extensionNames = extensionsList.split(","); + + /* + * To avoid of a test interference with an extension objects, create them in a separate schema. Of course, + * they must be truly relocatable. + */ + globalState.executeStatement(new SQLQueryAdapter("CREATE SCHEMA extensions;", true)); + for (int i = 0; i < extensionNames.length; i++) { + globalState.executeStatement(new SQLQueryAdapter( + "CREATE EXTENSION " + extensionNames[i] + " WITH SCHEMA extensions;", true)); } } + } - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - PostgresProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); - } - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - // TODO: transactions broke during refactoring - // catch (Throwable t) { - // if (t.getMessage().contains("current transaction is aborted")) { - // manager.execute(new QueryAdapter("ABORT")); - // globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); - // } else { - // System.err.println(query.getQueryString()); - // throw t; - // } - // } - se.executeStatements(); - manager.incrementCreateDatabase(); - manager.execute(new QueryAdapter("COMMIT")); - globalState.setSchema(PostgresSchema.fromConnection(con, databaseName)); + @Override + public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == PostgresOracleFactory.PQS)) { + generateOnlyKnown = true; + } - manager.execute(new QueryAdapter("SET SESSION statement_timeout = 5000;\n")); + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + entryPath = "/test"; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; + // trim URL to exclude "jdbc:" + if (entryURL.startsWith("jdbc:")) { + entryURL = entryURL.substring(5); + } + String entryDatabaseName = entryPath.substring(1); + databaseName = globalState.getDatabaseName(); - List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (SQLException e1) { - throw new AssertionError(e1); + try { + URI uri = new URI(entryURL); + String userInfoURI = uri.getUserInfo(); + String pathURI = uri.getPath(); + if (userInfoURI != null) { + // username and password specified in URL take precedence + if (userInfoURI.contains(":")) { + String[] userInfo = userInfoURI.split(":", 2); + username = userInfo[0]; + password = userInfo[1]; + } else { + username = userInfoURI; + password = null; + } + int userInfoIndex = entryURL.indexOf(userInfoURI); + String preUserInfo = entryURL.substring(0, userInfoIndex); + String postUserInfo = entryURL.substring(userInfoIndex + userInfoURI.length() + 1); + entryURL = preUserInfo + postUserInfo; } - }).collect(Collectors.toList()); - CompositeTestOracle oracle = new CompositeTestOracle(oracles); - - for (int i = 0; i < options.getNrQueries(); i++) { - try { - oracle.check(); - } catch (IgnoreMeException e) { - continue; + if (pathURI != null) { + entryPath = pathURI; } - manager.incrementSelectQueryCount(); + if (host == null) { + host = uri.getHost(); + } + if (port == MainOptions.NO_SET_PORT) { + port = uri.getPort(); + } + entryURL = String.format("%s://%s:%d/%s", uri.getScheme(), host, port, entryDatabaseName); + } catch (URISyntaxException e) { + throw new AssertionError(e); } + Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - } + String dropCommand = "DROP DATABASE"; + boolean forceDrop = Randomly.getBoolean(); + if (forceDrop) { + dropCommand += " FORCE"; + } + dropCommand += " IF EXISTS " + databaseName; - @Override - public Connection createDatabase(PostgresGlobalState globalState) throws SQLException { - String url = "jdbc:postgresql://localhost:5432/test"; - String databaseName = globalState.getDatabaseName(); - Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); - globalState.getState().statements.add(new QueryAdapter("\\c test;")); - globalState.getState().statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName)); - String createDatabaseCommand = getCreateDatabaseCommand(databaseName, con, globalState); - globalState.getState().statements.add(new QueryAdapter(createDatabaseCommand)); - globalState.getState().statements.add(new QueryAdapter("\\c " + databaseName)); + globalState.getState().logStatement(dropCommand + ";"); try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + s.execute(dropCommand); + } catch (SQLException e) { + // If force fails, fall back to regular drop + if (forceDrop) { + String fallbackDrop = "DROP DATABASE IF EXISTS " + databaseName; + globalState.getState().logStatement(fallbackDrop + ";"); + try (Statement s = con.createStatement()) { + s.execute(fallbackDrop); + } + } else { + throw e; + } } + + // Create database section + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand + ";"); try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); } con.close(); - con = DriverManager.getConnection("jdbc:postgresql://localhost:5432/" + databaseName, - globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - return con; + int databaseIndex = entryURL.indexOf(entryDatabaseName); + String preDatabaseName = entryURL.substring(0, databaseIndex); + String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); + testURL = preDatabaseName + databaseName + postDatabaseName; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + + con = DriverManager.getConnection("jdbc:" + testURL, username, password); + return new SQLConnection(con); } - private String getCreateDatabaseCommand(String databaseName, Connection con, GlobalState state) { + protected void readFunctions(PostgresGlobalState globalState) throws SQLException { + SQLQueryAdapter query = new SQLQueryAdapter("SELECT proname, provolatile FROM pg_proc;"); + SQLancerResultSet rs = query.executeAndGet(globalState); + while (rs.next()) { + String functionName = rs.getString(1); + Character functionType = rs.getString(2).charAt(0); + globalState.addFunctionAndType(functionName, functionType); + } + } + + protected void createTables(PostgresGlobalState globalState, int numTables) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = PostgresTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + globalState.executeStatement(createTable); + } catch (IgnoreMeException e) { + + } + } + } + + protected void prepareTables(PostgresGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + PostgresProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 5000;\n")); + } + + private String getCreateDatabaseCommand(PostgresGlobalState state) { StringBuilder sb = new StringBuilder(); sb.append("CREATE DATABASE " + databaseName + " "); - if (Randomly.getBoolean() && ((PostgresOptions) state.getDmbsSpecificOptions()).testCollations) { + if (((PostgresOptions) state.getDbmsSpecificOptions()).testCollations) { if (Randomly.getBoolean()) { - sb.append("WITH ENCODING '"); - sb.append(Randomly.fromOptions("utf8")); - sb.append("' "); - } - for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { if (Randomly.getBoolean()) { - globalState = new PostgresGlobalState(); - globalState.setConnection(con); - sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(globalState.getCollates()))); + sb.append("WITH ENCODING '"); + sb.append(Randomly.fromOptions("utf8")); + sb.append("' "); + } + if (Randomly.getBoolean() && !state.getCollates().isEmpty()) { + sb.append(String.format(" LOCALE = '%s' ", Randomly.fromList(state.getCollates()))); + } else { + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } + } } + sb.append(" TEMPLATE template0"); } - sb.append(" TEMPLATE template0"); + } else { + sb.append("WITH ENCODING 'UTF8' TEMPLATE template0"); } return sb.toString(); } @@ -317,43 +397,74 @@ public String getDBMSName() { } @Override - public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { - StringBuilder sb = new StringBuilder(); - PostgresStateToReproduce specificState = (PostgresStateToReproduce) state; - if (specificState.getRandomRowValues() != null) { - List columnList = specificState.getRandomRowValues().keySet().stream() - .collect(Collectors.toList()); - List tableList = columnList.stream().map(c -> c.getTable()).distinct().sorted() - .collect(Collectors.toList()); - for (PostgresTable t : tableList) { - sb.append("-- " + t.getName() + "\n"); - List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) - .collect(Collectors.toList()); - for (PostgresColumn c : columnsForTable) { - sb.append("--\t"); - sb.append(c); - sb.append("="); - sb.append(specificState.getRandomRowValues().get(c)); - sb.append("\n"); - } - } - sb.append("expected values: \n"); - PostgresExpression whereClause = ((PostgresStateToReproduce) state).getWhereClause(); - if (whereClause != null) { - sb.append(PostgresVisitor.asExpectedValues(whereClause).replace("\n", "\n-- ")); + public String getQueryPlan(String selectStr, PostgresGlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); } } - try { - writer.write(sb.toString()); - writer.flush(); - } catch (IOException e) { - throw new AssertionError(); + SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr), null); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + while (rs.next()) { + queryPlan += rs.getString(1); + } + } catch (SQLException | AssertionError e) { + queryPlan = ""; } + return formatQueryPlan(queryPlan); } @Override - public StateToReproduce getStateToReproduce(String databaseName) { - return new PostgresStateToReproduce(databaseName); + protected double[] initializeWeightedAverageReward() { + return new double[PostgresProvider.Action.values().length]; + } + + @Override + protected void executeMutator(int index, PostgresGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = PostgresProvider.Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + + @Override + protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (PostgresSchema.PostgresTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = PostgresInsertGenerator.insertRows(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + + public String formatQueryPlan(String queryPlan) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(queryPlan).get(0).get("Plan"); + // Extract nodes using BFS algorithm + List nodeTypes = extractNodeTypesIterative(root); + return String.join(" ", nodeTypes); + } + + // BFS algorithm for traversing the Json Query Plan + private static List extractNodeTypesIterative(JsonNode root) { + List result = new ArrayList<>(); + Queue queue = new LinkedList<>(); + queue.add(root); + while (!queue.isEmpty()) { + JsonNode node = queue.poll(); + if (node.has("Node Type")) { + result.add(node.get("Node Type").asText()); + } + if (node.has("Plans") && node.get("Plans").isArray()) { + for (JsonNode plan : node.get("Plans")) { + queue.add(plan); + } + } + } + return result; } } diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index d99c6f223..82937557c 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -1,44 +1,48 @@ package sqlancer.postgres; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; + +import org.postgresql.util.PSQLException; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.StateToReproduce.PostgresStateToReproduce; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; import sqlancer.postgres.ast.PostgresConstant; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; -public class PostgresSchema { +public class PostgresSchema extends AbstractSchema { - private final List databaseTables; private final String databaseName; public enum PostgresDataType { INT, BOOLEAN, TEXT, DECIMAL, FLOAT, REAL, RANGE, MONEY, BIT, INET; public static PostgresDataType getRandomType() { - List dataTypes = Arrays.asList(values()); + List dataTypes = new ArrayList<>(Arrays.asList(values())); if (PostgresProvider.generateOnlyKnown) { dataTypes.remove(PostgresDataType.DECIMAL); dataTypes.remove(PostgresDataType.FLOAT); dataTypes.remove(PostgresDataType.REAL); dataTypes.remove(PostgresDataType.INET); + dataTypes.remove(PostgresDataType.RANGE); + dataTypes.remove(PostgresDataType.MONEY); + dataTypes.remove(PostgresDataType.BIT); } return Randomly.fromList(dataTypes); } @@ -50,6 +54,10 @@ public PostgresColumn(String name, PostgresDataType columnType) { super(name, null, columnType); } + public static PostgresColumn createDummy(String name) { + return new PostgresColumn(name, PostgresDataType.INT); + } + } public static class PostgresTables extends AbstractTables { @@ -58,7 +66,7 @@ public PostgresTables(List tables) { super(tables); } - public PostgresRowValue getRandomRowValue(Connection con, PostgresStateToReproduce state) throws SQLException { + public PostgresRowValue getRandomRowValue(SQLConnection con) throws SQLException { String randomRow = String.format("SELECT %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + @@ -68,7 +76,7 @@ public PostgresRowValue getRandomRowValue(Connection con, PostgresStateToReprodu try (Statement s = con.createStatement()) { ResultSet randomRowValues = s.executeQuery(randomRow); if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow + "\n" + state); + throw new AssertionError("could not find random row! " + randomRow + "\n"); } for (int i = 0; i < getColumns().size(); i++) { PostgresColumn column = getColumns().get(i); @@ -89,21 +97,22 @@ public PostgresRowValue getRandomRowValue(Connection con, PostgresStateToReprodu constant = PostgresConstant.createTextConstant(randomRowValues.getString(columnIndex)); break; default: - throw new AssertionError(column.getType()); + throw new IgnoreMeException(); } } values.put(column, constant); } assert !randomRowValues.next(); - state.randomRowValues = values; return new PostgresRowValue(this, values); + } catch (PSQLException e) { + throw new IgnoreMeException(); } } } - private static PostgresDataType getColumnType(String typeString) { + public static PostgresDataType getColumnType(String typeString) { switch (typeString) { case "smallint": case "integer": @@ -115,6 +124,13 @@ private static PostgresDataType getColumnType(String typeString) { case "character": case "character varying": case "name": + case "regclass": + case "regnamespace": + case "regrole": + case "regtype": + case "regproc": + case "regprocedure": + case "regoper": return PostgresDataType.TEXT; case "numeric": return PostgresDataType.DECIMAL; @@ -136,60 +152,16 @@ private static PostgresDataType getColumnType(String typeString) { } } - public static class PostgresRowValue { - - private final PostgresTables tables; - private final Map values; - - PostgresRowValue(PostgresTables tables, Map values) { - this.tables = tables; - this.values = values; - } + public static class PostgresRowValue extends AbstractRowValue { - public PostgresTables getTable() { - return tables; - } - - public Map getValues() { - return values; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - int i = 0; - for (PostgresColumn c : tables.getColumns()) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(values.get(c)); - } - return sb.toString(); - } - - public String getRowValuesAsString() { - List columnsToCheck = tables.getColumns(); - return getRowValuesAsString(columnsToCheck); - } - - public String getRowValuesAsString(List columnsToCheck) { - StringBuilder sb = new StringBuilder(); - Map expectedValues = getValues(); - for (int i = 0; i < columnsToCheck.size(); i++) { - if (i != 0) { - sb.append(", "); - } - PostgresConstant expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); - PostgresToStringVisitor visitor = new PostgresToStringVisitor(); - visitor.visit(expectedColumnValue); - sb.append(visitor.get()); - } - return sb.toString(); + protected PostgresRowValue(PostgresTables tables, Map values) { + super(tables, values); } } - public static class PostgresTable extends AbstractTable { + public static class PostgresTable + extends AbstractRelationalTable { public enum TableType { STANDARD, TEMPORARY @@ -198,6 +170,7 @@ public enum TableType { private final TableType tableType; private final List statistics; private final boolean isInsertable; + private final boolean isPartitioned; public PostgresTable(String tableName, List columns, List indexes, TableType tableType, List statistics, boolean isView, boolean isInsertable) { @@ -205,6 +178,18 @@ public PostgresTable(String tableName, List columns, List columns, List indexes, + TableType tableType, List statistics, boolean isView, boolean isInsertable, + boolean isPartitioned) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + this.isPartitioned = isPartitioned; } public List getStatistics() { @@ -219,6 +204,10 @@ public boolean isInsertable() { return isInsertable; } + public boolean isPartitioned() { + return isPartitioned; + } + } public static final class PostgresStatisticsObject { @@ -254,28 +243,28 @@ public String getIndexName() { } - public static PostgresSchema fromConnection(Connection con, String databaseName) throws SQLException { - Exception ex = null; + public static PostgresSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { try { List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery( - "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { + "SELECT t.table_name, t.table_schema, t.table_type, t.is_insertable_into, c.relkind FROM information_schema.tables t JOIN pg_class c ON c.relname = t.table_name JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = t.table_schema WHERE t.table_schema='public' OR t.table_schema LIKE 'pg_temp_%' ORDER BY t.table_name;")) { while (rs.next()) { String tableName = rs.getString("table_name"); String tableTypeSchema = rs.getString("table_schema"); boolean isInsertable = rs.getBoolean("is_insertable_into"); + boolean isPartitioned = "p".equals(rs.getString("relkind")); // TODO: also check insertable // TODO: insert into view? - boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || - // tableTypeStr.contains("LOCAL TEMPORARY") && - // !isInsertable; + boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || + // tableTypeStr.contains("LOCAL TEMPORARY") && + // !isInsertable; PostgresTable.TableType tableType = getTableType(tableTypeSchema); List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); List statistics = getStatistics(con); PostgresTable t = new PostgresTable(tableName, databaseColumns, indexes, tableType, statistics, - isView, isInsertable); + isView, isInsertable, isPartitioned); for (PostgresColumn c : databaseColumns) { c.setTable(t); } @@ -285,15 +274,14 @@ public static PostgresSchema fromConnection(Connection con, String databaseName) } return new PostgresSchema(databaseTables, databaseName); } catch (SQLIntegrityConstraintViolationException e) { - ex = e; + throw new AssertionError(e); } - throw new AssertionError(ex); } - private static List getStatistics(Connection con) throws SQLException { + protected static List getStatistics(SQLConnection con) throws SQLException { List statistics = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext;")) { + try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext ORDER BY stxname;")) { while (rs.next()) { statistics.add(new PostgresStatisticsObject(rs.getString("stxname"))); } @@ -302,7 +290,7 @@ private static List getStatistics(Connection con) thro return statistics; } - private static PostgresTable.TableType getTableType(String tableTypeStr) throws AssertionError { + protected static PostgresTable.TableType getTableType(String tableTypeStr) throws AssertionError { PostgresTable.TableType tableType; if (tableTypeStr.contentEquals("public")) { tableType = TableType.STANDARD; @@ -314,30 +302,28 @@ private static PostgresTable.TableType getTableType(String tableTypeStr) throws return tableType; } - private static List getIndexes(Connection con, String tableName) throws SQLException { + protected static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s - .executeQuery(String.format("SELECT indexname FROM pg_indexes WHERE tablename='%s';", tableName))) { + try (ResultSet rs = s.executeQuery(String + .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' ORDER BY indexname;", tableName))) { while (rs.next()) { String indexName = rs.getString("indexname"); - if (indexName.length() != 2) { - // FIXME: implement cleanly - continue; // skip internal indexes + if (DBMSCommon.matchesIndexName(indexName)) { + indexes.add(PostgresIndex.create(indexName)); } - indexes.add(PostgresIndex.create(indexName)); } } } return indexes; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + protected static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s .executeQuery("select column_name, data_type from INFORMATION_SCHEMA.COLUMNS where table_name = '" - + tableName + "'")) { + + tableName + "' ORDER BY column_name")) { while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); @@ -350,46 +336,16 @@ private static List getTableColumns(Connection con, String table } public PostgresSchema(List databaseTables, String databaseName) { - this.databaseTables = Collections.unmodifiableList(databaseTables); + super(databaseTables); this.databaseName = databaseName; } - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - for (PostgresTable t : getDatabaseTables()) { - sb.append(t + "\n"); - } - return sb.toString(); - } - - public PostgresTable getRandomTable() { - return Randomly.fromList(getDatabaseTables()); - } - public PostgresTables getRandomTableNonEmptyTables() { - return new PostgresTables(Randomly.nonEmptySubset(databaseTables)); - } - - public List getDatabaseTables() { - return databaseTables; - } - - public List getDatabaseTablesRandomSubsetNotEmpty() { - return Randomly.nonEmptySubset(databaseTables); + return new PostgresTables(Randomly.nonEmptySubset(getDatabaseTables())); } public String getDatabaseName() { return databaseName; } - public PostgresTable getRandomTable(Function f) { - List relevantTables = databaseTables.stream().filter(t -> f.apply(t)) - .collect(Collectors.toList()); - if (relevantTables.isEmpty()) { - throw new IgnoreMeException(); - } - return Randomly.fromList(relevantTables); - } - } diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 66de6c94f..87bd3c429 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -1,13 +1,18 @@ package sqlancer.postgres; +import java.util.List; import java.util.Optional; import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; @@ -15,6 +20,7 @@ import sqlancer.postgres.ast.PostgresInOperation; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -22,8 +28,12 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; -import sqlancer.visitor.ToStringVisitor; +import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; public final class PostgresToStringVisitor extends ToStringVisitor implements PostgresVisitor { @@ -42,6 +52,11 @@ public String get() { return sb.toString(); } + @Override + public void visit(PostgresColumnReference column) { + sb.append(column.getColumn().getFullQualifiedName()); + } + @Override public void visit(PostgresPostfixOperation op) { sb.append("("); @@ -75,6 +90,19 @@ public void visit(PostgresFromTable from) { } } + @Override + public void visit(PostgresSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(PostgresTableReference ref) { + sb.append(ref.getTable().getName()); + } + @Override public void visit(PostgresSelect s) { sb.append("SELECT "); @@ -126,7 +154,7 @@ public void visit(PostgresSelect s) { throw new AssertionError(j.getType()); } sb.append(" "); - sb.append(j.getTable().getName()); + visit(j.getTableReference()); if (j.getType() != PostgresJoinType.CROSS) { sb.append(" ON "); visit(j.getOnClause()); @@ -137,7 +165,7 @@ public void visit(PostgresSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } @@ -146,9 +174,9 @@ public void visit(PostgresSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); @@ -162,10 +190,9 @@ public void visit(PostgresSelect s) { } @Override - public void visit(PostgresOrderByTerm op) { - visit(op.getExpr()); - sb.append(" "); - sb.append(op.getOrder()); + public void visit(PostgresOrderByTerm term) { + visit(term.getExpr()); + sb.append(term.isAscending() ? " ASC" : " DESC"); } @Override @@ -328,4 +355,48 @@ public void visit(PostgresCollate op) { sb.append(")"); } + @Override + public void visit(PostgresBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + @Override + public void visit(PostgresLikeOperation op) { + super.visit((BinaryOperation) op); + } + + @Override + @SuppressWarnings("unchecked") + public void visit(PostgresWindowFunction windowFunction) { + sb.append(windowFunction.getFunctionName()); + sb.append("("); + visit(windowFunction.getArguments()); + sb.append(") OVER ("); + + WindowSpecification spec = windowFunction.getWindowSpec(); + if (!spec.getPartitionBy().isEmpty()) { + sb.append("PARTITION BY "); + visit(spec.getPartitionBy()); + } + + if (!spec.getOrderBy().isEmpty()) { + if (!spec.getPartitionBy().isEmpty()) { + sb.append(" "); + } + sb.append("ORDER BY "); + visit((List) (List) spec.getOrderBy()); + } + + if (spec.getFrame() != null) { + sb.append(" "); + WindowFrame frame = spec.getFrame(); + sb.append(frame.getType().getSQL()); + sb.append(" BETWEEN "); + visit(frame.getStartExpr()); + sb.append(" AND "); + visit(frame.getEndExpr()); + } + + sb.append(")"); + } } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 06f990fc4..d5bf71e81 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -6,13 +6,16 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresFunction; import sqlancer.postgres.ast.PostgresInOperation; +import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -20,7 +23,10 @@ import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.gen.PostgresExpressionGenerator; public interface PostgresVisitor { @@ -31,6 +37,10 @@ public interface PostgresVisitor { void visit(PostgresColumnValue c); + void visit(PostgresColumnReference c); + + void visit(PostgresTableReference tb); + void visit(PostgresPrefixOperation op); void visit(PostgresSelect op); @@ -57,6 +67,14 @@ public interface PostgresVisitor { void visit(PostgresFromTable from); + void visit(PostgresSubquery subquery); + + void visit(PostgresBinaryLogicalOperation op); + + void visit(PostgresLikeOperation op); + + void visit(PostgresWindowFunction windowFunction); + default void visit(PostgresExpression expression) { if (expression instanceof PostgresConstant) { visit((PostgresConstant) expression); @@ -90,6 +108,16 @@ default void visit(PostgresExpression expression) { visit((PostgresCollate) expression); } else if (expression instanceof PostgresFromTable) { visit((PostgresFromTable) expression); + } else if (expression instanceof PostgresSubquery) { + visit((PostgresSubquery) expression); + } else if (expression instanceof PostgresLikeOperation) { + visit((PostgresLikeOperation) expression); + } else if (expression instanceof PostgresColumnReference) { + visit((PostgresColumnReference) expression); + } else if (expression instanceof PostgresTableReference) { + visit((PostgresTableReference) expression); + } else if (expression instanceof PostgresWindowFunction) { + visit((PostgresWindowFunction) expression); } else { throw new AssertionError(expression); } @@ -107,13 +135,6 @@ static String asExpectedValues(PostgresExpression expr) { return v.get(); } - static String getExpressionAsString(PostgresGlobalState globalState, PostgresDataType type) { - PostgresExpression expression = PostgresExpressionGenerator.generateExpression(globalState, type); - PostgresToStringVisitor visitor = new PostgresToStringVisitor(); - visitor.visit(expression); - return visitor.get(); - } - static String getExpressionAsString(PostgresGlobalState globalState, PostgresDataType type, List columns) { PostgresExpression expression = PostgresExpressionGenerator.generateExpression(globalState, columns, type); diff --git a/src/sqlancer/postgres/ast/PostgresAggregate.java b/src/sqlancer/postgres/ast/PostgresAggregate.java index 9159cb191..c266e92ad 100644 --- a/src/sqlancer/postgres/ast/PostgresAggregate.java +++ b/src/sqlancer/postgres/ast/PostgresAggregate.java @@ -5,12 +5,12 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.ast.FunctionNode; +import sqlancer.common.ast.FunctionNode; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate.PostgresAggregateFunction; /** - * @see https://www.sqlite.org/lang_aggfunc.html + * @see Built-in Aggregate Functions */ public class PostgresAggregate extends FunctionNode implements PostgresExpression { @@ -28,14 +28,6 @@ public enum PostgresAggregateFunction { this.supportedReturnTypes = supportedReturnTypes.clone(); } - public static PostgresAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public static PostgresAggregateFunction getRandom(PostgresDataType type) { - return Randomly.fromOptions(values()); - } - public List getTypes(PostgresDataType returnType) { return Arrays.asList(returnType); } diff --git a/src/sqlancer/postgres/ast/PostgresAlias.java b/src/sqlancer/postgres/ast/PostgresAlias.java index 38247aeec..9f46084b1 100644 --- a/src/sqlancer/postgres/ast/PostgresAlias.java +++ b/src/sqlancer/postgres/ast/PostgresAlias.java @@ -1,6 +1,6 @@ package sqlancer.postgres.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class PostgresAlias implements UnaryOperation, PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java index 95639f34e..a36173743 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryArithmeticOperation.java @@ -3,8 +3,8 @@ import java.util.function.BinaryOperator; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresBinaryArithmeticOperation.PostgresBinaryOperator; @@ -48,12 +48,10 @@ public PostgresConstant apply(PostgresConstant left, PostgresConstant right) { } }, - // TODO no implementation EXPONENTIATION("^") { @Override public PostgresConstant apply(PostgresConstant left, PostgresConstant right) { - // return applyBitOperation(left, right, (l, r) -> (long) Math.pow(l, r)); - throw new AssertionError(); + return null; } }; @@ -97,6 +95,9 @@ public PostgresBinaryArithmeticOperation(PostgresExpression left, PostgresExpres public PostgresConstant getExpectedValue() { PostgresConstant leftExpected = getLeft().getExpectedValue(); PostgresConstant rightExpected = getRight().getExpectedValue(); + if (leftExpected == null || rightExpected == null) { + return null; + } return getOp().apply(leftExpected, rightExpected); } diff --git a/src/sqlancer/postgres/ast/PostgresBinaryBitOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryBitOperation.java index 4ef9c690a..d111386da 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryBitOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryBitOperation.java @@ -1,8 +1,8 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresBinaryBitOperation.PostgresBinaryBitOperator; diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index ae4f9a16a..b77060dfd 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -1,8 +1,9 @@ package sqlancer.postgres.ast; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresBinaryComparisonOperation.PostgresBinaryComparisonOperator; @@ -123,7 +124,12 @@ public PostgresBinaryComparisonOperation(PostgresExpression left, PostgresExpres @Override public PostgresConstant getExpectedValue() { - return getOp().getExpectedValue(getLeft().getExpectedValue(), getRight().getExpectedValue()); + PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); + PostgresConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + throw new IgnoreMeException(); + } + return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); } @Override diff --git a/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java index 63bc39ea6..9c104670e 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryLogicalOperation.java @@ -1,8 +1,8 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresBinaryLogicalOperation.BinaryLogicalOperator; @@ -77,7 +77,12 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - return getOp().apply(getLeft().getExpectedValue(), getRight().getExpectedValue()); + PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); + PostgresConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); } } diff --git a/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java index 5f0f246b5..b2594522a 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java @@ -1,8 +1,8 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresBinaryRangeOperation extends BinaryNode implements PostgresExpression { @@ -10,7 +10,7 @@ public class PostgresBinaryRangeOperation extends BinaryNode private final String op; public enum PostgresBinaryRangeOperator implements Operator { - UNION("*"), INTERSECTION("*"), DIFFERENCE("-"); + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); private final String textRepresentation; diff --git a/src/sqlancer/postgres/ast/PostgresCastOperation.java b/src/sqlancer/postgres/ast/PostgresCastOperation.java index dcd2d3e56..521b2cd97 100644 --- a/src/sqlancer/postgres/ast/PostgresCastOperation.java +++ b/src/sqlancer/postgres/ast/PostgresCastOperation.java @@ -23,7 +23,11 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - return expression.getExpectedValue().cast(type.getDataType()); + PostgresConstant expectedValue = expression.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type.getDataType()); } public PostgresExpression getExpression() { diff --git a/src/sqlancer/postgres/ast/PostgresColumnReference.java b/src/sqlancer/postgres/ast/PostgresColumnReference.java new file mode 100644 index 000000000..0d835bf89 --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresColumnReference.java @@ -0,0 +1,15 @@ +package sqlancer.postgres.ast; + +import sqlancer.postgres.PostgresSchema.PostgresColumn; + +public class PostgresColumnReference implements PostgresExpression { + private final PostgresColumn c; + + public PostgresColumnReference(PostgresColumn c) { + this.c = c; + } + + public PostgresColumn getColumn() { + return c; + } +} diff --git a/src/sqlancer/postgres/ast/PostgresColumnValue.java b/src/sqlancer/postgres/ast/PostgresColumnValue.java index ccdd5137d..0974ebea8 100644 --- a/src/sqlancer/postgres/ast/PostgresColumnValue.java +++ b/src/sqlancer/postgres/ast/PostgresColumnValue.java @@ -23,10 +23,6 @@ public PostgresConstant getExpectedValue() { return expectedValue; } - public static PostgresColumnValue create(PostgresColumn c) { - return new PostgresColumnValue(c, null); - } - public static PostgresColumnValue create(PostgresColumn c, PostgresConstant expected) { return new PostgresColumnValue(c, expected); } diff --git a/src/sqlancer/postgres/ast/PostgresConcatOperation.java b/src/sqlancer/postgres/ast/PostgresConcatOperation.java index b2e8f6542..cbbe59085 100644 --- a/src/sqlancer/postgres/ast/PostgresConcatOperation.java +++ b/src/sqlancer/postgres/ast/PostgresConcatOperation.java @@ -1,6 +1,6 @@ package sqlancer.postgres.ast; -import sqlancer.ast.BinaryNode; +import sqlancer.common.ast.BinaryNode; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresConcatOperation extends BinaryNode implements PostgresExpression { @@ -16,11 +16,16 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - if (getLeft().getExpectedValue().isNull() || getRight().getExpectedValue().isNull()) { + PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); + PostgresConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + if (leftExpectedValue.isNull() || rightExpectedValue.isNull()) { return PostgresConstant.createNullConstant(); } - String leftStr = getLeft().getExpectedValue().cast(PostgresDataType.TEXT).getUnquotedTextRepresentation(); - String rightStr = getRight().getExpectedValue().cast(PostgresDataType.TEXT).getUnquotedTextRepresentation(); + String leftStr = leftExpectedValue.cast(PostgresDataType.TEXT).getUnquotedTextRepresentation(); + String rightStr = rightExpectedValue.cast(PostgresDataType.TEXT).getUnquotedTextRepresentation(); return PostgresConstant.createTextConstant(leftStr + rightStr); } diff --git a/src/sqlancer/postgres/ast/PostgresConstant.java b/src/sqlancer/postgres/ast/PostgresConstant.java index d5bc3b671..c898ff95e 100644 --- a/src/sqlancer/postgres/ast/PostgresConstant.java +++ b/src/sqlancer/postgres/ast/PostgresConstant.java @@ -75,7 +75,7 @@ public PostgresConstant cast(PostgresDataType type) { case TEXT: return PostgresConstant.createTextConstant(value ? "true" : "false"); default: - throw new AssertionError(); + return null; } } @@ -212,7 +212,7 @@ public PostgresConstant cast(PostgresDataType type) { case TEXT: return this; default: - throw new AssertionError(this); + return null; } } @@ -307,7 +307,7 @@ public PostgresConstant cast(PostgresDataType type) { case TEXT: return PostgresConstant.createTextConstant(String.valueOf(val)); default: - throw new AssertionError(type); + return null; } } @@ -390,24 +390,22 @@ public abstract static class PostgresConstantBase extends PostgresConstant { @Override public String getUnquotedTextRepresentation() { - throw new AssertionError(); + return null; } @Override public PostgresConstant isEquals(PostgresConstant rightVal) { - throw new AssertionError(); + return null; } @Override protected PostgresConstant isLessThan(PostgresConstant rightVal) { - throw new AssertionError(); - + return null; } @Override public PostgresConstant cast(PostgresDataType type) { - throw new AssertionError(); - + return null; } } diff --git a/src/sqlancer/postgres/ast/PostgresExpression.java b/src/sqlancer/postgres/ast/PostgresExpression.java index ed4516ffa..433e2207c 100644 --- a/src/sqlancer/postgres/ast/PostgresExpression.java +++ b/src/sqlancer/postgres/ast/PostgresExpression.java @@ -1,14 +1,16 @@ package sqlancer.postgres.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; -public interface PostgresExpression { +public interface PostgresExpression extends Expression { default PostgresDataType getExpressionType() { - throw new AssertionError("operator does not support PQS evaluation!"); + return null; } default PostgresConstant getExpectedValue() { - throw new AssertionError("operator does not support PQS evaluation!"); + return null; } } diff --git a/src/sqlancer/postgres/ast/PostgresFunction.java b/src/sqlancer/postgres/ast/PostgresFunction.java index a5bbc25b6..5fe2968ab 100644 --- a/src/sqlancer/postgres/ast/PostgresFunction.java +++ b/src/sqlancer/postgres/ast/PostgresFunction.java @@ -1,6 +1,5 @@ package sqlancer.postgres.ast; -import sqlancer.Randomly; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresFunction implements PostgresExpression { @@ -225,29 +224,17 @@ public PostgresDataType[] getRandomTypes(int nr) { return types; } - public PostgresDataType[] getType(int nr, PostgresDataType type) { - PostgresDataType[] types = new PostgresDataType[nr]; - for (int i = 0; i < types.length; i++) { - types[i] = type; - } - return types; - } - PostgresFunctionWithResult(int nrArgs, String functionName) { this.nrArgs = nrArgs; this.functionName = functionName; this.variadic = false; } - PostgresFunctionWithResult(int nrArgs, String functionName, boolean variadic) { - this.nrArgs = nrArgs; - this.functionName = functionName; - this.variadic = variadic; - } - /** * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number * of arguments is returned. + * + * @return the number of arguments */ public int getNrArgs() { return nrArgs; @@ -255,10 +242,6 @@ public int getNrArgs() { public abstract PostgresConstant apply(PostgresConstant[] evaluatedArgs, PostgresExpression... args); - public static PostgresFunctionWithResult getRandomFunction() { - return Randomly.fromOptions(values()); - } - @Override public String toString() { return functionName; @@ -284,10 +267,15 @@ public boolean checkArguments(PostgresExpression... constants) { @Override public PostgresConstant getExpectedValue() { - assert functionWithKnownResult != null; + if (functionWithKnownResult == null) { + return null; + } PostgresConstant[] constants = new PostgresConstant[args.length]; for (int i = 0; i < constants.length; i++) { constants[i] = args[i].getExpectedValue(); + if (constants[i] == null) { + return null; + } } return functionWithKnownResult.apply(constants, args); } diff --git a/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java b/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java index 3357a7db5..287f46784 100644 --- a/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java +++ b/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java @@ -19,11 +19,11 @@ public enum PostgresFunctionWithUnknownResult { TEXT("text", PostgresDataType.TEXT, PostgresDataType.INET), INET_SAME_FAMILY("inet_same_family", PostgresDataType.BOOLEAN, PostgresDataType.INET, PostgresDataType.INET), - // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE + // https://www.postgresql.org/docs/13/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE // PG_RELOAD_CONF("pg_reload_conf", PostgresDataType.BOOLEAN), // too much output // PG_ROTATE_LOGFILE("pg_rotate_logfile", PostgresDataType.BOOLEAN), prints warning - // https://www.postgresql.org/docs/devel/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE + // https://www.postgresql.org/docs/13/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE CURRENT_DATABASE("current_database", PostgresDataType.TEXT), // name // CURRENT_QUERY("current_query", PostgresDataType.TEXT), // can generate false positives CURRENT_SCHEMA("current_schema", PostgresDataType.TEXT), // name @@ -87,7 +87,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx TO_HEX("to_hex", PostgresDataType.INT, PostgresDataType.TEXT), TRANSLATE("translate", PostgresDataType.TEXT, PostgresDataType.TEXT, PostgresDataType.TEXT, PostgresDataType.TEXT), // mathematical functions - // https://www.postgresql.org/docs/9.5/functions-math.html + // https://www.postgresql.org/docs/13/functions-math.html ABS("abs", PostgresDataType.REAL, PostgresDataType.REAL), CBRT("cbrt", PostgresDataType.REAL, PostgresDataType.REAL), CEILING("ceiling", PostgresDataType.REAL), // DEGREES("degrees", PostgresDataType.REAL), EXP("exp", PostgresDataType.REAL), LN("ln", PostgresDataType.REAL), @@ -98,7 +98,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx FLOOR("floor", PostgresDataType.REAL), // trigonometric functions - complete - // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE + // https://www.postgresql.org/docs/13/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE ACOS("acos", PostgresDataType.REAL), // ACOSD("acosd", PostgresDataType.REAL), // ASIN("asin", PostgresDataType.REAL), // @@ -117,7 +117,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx TAND("tand", PostgresDataType.REAL), // // hyperbolic functions - complete - // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-HYP-TABLE + // https://www.postgresql.org/docs/13/functions-math.html#FUNCTIONS-MATH-HYP-TABLE SINH("sinh", PostgresDataType.REAL), // COSH("cosh", PostgresDataType.REAL), // TANH("tanh", PostgresDataType.REAL), // @@ -125,12 +125,12 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx ACOSH("acosh", PostgresDataType.REAL), // ATANH("atanh", PostgresDataType.REAL), // - // https://www.postgresql.org/docs/devel/functions-binarystring.html + // https://www.postgresql.org/docs/13/functions-binarystring.html GET_BIT("get_bit", PostgresDataType.INT, PostgresDataType.TEXT, PostgresDataType.INT), GET_BYTE("get_byte", PostgresDataType.INT, PostgresDataType.TEXT, PostgresDataType.INT), // range functions - // https://www.postgresql.org/docs/devel/functions-range.html#RANGE-FUNCTIONS-TABLE + // https://www.postgresql.org/docs/13/functions-range.html#RANGE-FUNCTIONS-TABLE RANGE_LOWER("lower", PostgresDataType.INT, PostgresDataType.RANGE), // RANGE_UPPER("upper", PostgresDataType.INT, PostgresDataType.RANGE), // RANGE_ISEMPTY("isempty", PostgresDataType.BOOLEAN, PostgresDataType.RANGE), // @@ -140,7 +140,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx RANGE_UPPER_INF("upper_inf", PostgresDataType.BOOLEAN, PostgresDataType.RANGE), // RANGE_MERGE("range_merge", PostgresDataType.RANGE, PostgresDataType.RANGE, PostgresDataType.RANGE), // - // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE + // https://www.postgresql.org/docs/13/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE GET_COLUMN_SIZE("get_column_size", PostgresDataType.INT, PostgresDataType.TEXT); // PG_DATABASE_SIZE("pg_database_size", PostgresDataType.INT, PostgresDataType.INT); // PG_SIZE_BYTES("pg_size_bytes", PostgresDataType.INT, PostgresDataType.TEXT); diff --git a/src/sqlancer/postgres/ast/PostgresInOperation.java b/src/sqlancer/postgres/ast/PostgresInOperation.java index 73bd8ec57..68f4b38cc 100644 --- a/src/sqlancer/postgres/ast/PostgresInOperation.java +++ b/src/sqlancer/postgres/ast/PostgresInOperation.java @@ -26,15 +26,23 @@ public List getListElements() { @Override public PostgresConstant getExpectedValue() { - if (expr.getExpectedValue().isNull()) { + PostgresConstant leftValue = expr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { return PostgresConstant.createNullConstant(); } boolean isNull = false; for (PostgresExpression expr : getListElements()) { - if (expr.getExpectedValue().isNull()) { + PostgresConstant rightExpectedValue = expr.getExpectedValue(); + if (rightExpectedValue == null) { + return null; + } + if (rightExpectedValue.isNull()) { isNull = true; - } else if (expr.getExpectedValue().isEquals(this.expr.getExpectedValue()).isBoolean() - && expr.getExpectedValue().isEquals(this.expr.getExpectedValue()).asBoolean()) { + } else if (rightExpectedValue.isEquals(this.expr.getExpectedValue()).isBoolean() + && rightExpectedValue.isEquals(this.expr.getExpectedValue()).asBoolean()) { return PostgresConstant.createBooleanConstant(isTrue); } } diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index 053541659..ef6dfab1f 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -1,10 +1,18 @@ package sqlancer.postgres.ast; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class PostgresJoin implements PostgresExpression { +public class PostgresJoin implements PostgresExpression, Join { public enum PostgresJoinType { INNER, LEFT, RIGHT, FULL, CROSS; @@ -13,20 +21,83 @@ public static PostgresJoinType getRandom() { return Randomly.fromOptions(values()); } + public static PostgresJoinType getRandomExcept(PostgresJoinType... exclude) { + PostgresJoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(PostgresJoinType[]::new); + return Randomly.fromOptions(values); + } + } - private final PostgresTable table; - private final PostgresExpression onClause; - private final PostgresJoinType type; + private final PostgresExpression tableReference; + private PostgresExpression onClause; + private PostgresJoinType type; + private final PostgresExpression leftTable; + private final PostgresExpression rightTable; - public PostgresJoin(PostgresTable table, PostgresExpression onClause, PostgresJoinType type) { - this.table = table; + public PostgresJoin(PostgresExpression tableReference, PostgresExpression onClause, PostgresJoinType type) { + this.tableReference = tableReference; this.onClause = onClause; this.type = type; + this.leftTable = null; + this.rightTable = null; + } + + public PostgresJoin(PostgresExpression leftTable, PostgresExpression rightTable, PostgresJoinType joinType, + PostgresExpression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.type = joinType; + this.onClause = whereCondition; + this.tableReference = null; + } + + public static PostgresJoin createJoin(PostgresExpression left, PostgresExpression right, PostgresJoinType type, + PostgresExpression onClause) { + if (type == PostgresJoinType.CROSS) { + return new PostgresJoin(left, right, type, null); + } else { + return new PostgresJoin(left, right, type, onClause); + } + } + + public static List getJoins(List tableList, + PostgresGlobalState globalState) { + // Clone Table to prevent the original list from being manipulated + List tbl = new ArrayList<>(tableList); + List joinExpressions = new ArrayList<>(); + while (tbl.size() >= 2 && Randomly.getBoolean()) { + PostgresTableReference left = (PostgresTableReference) tbl.remove(0); + PostgresTableReference right = (PostgresTableReference) tbl.remove(0); + List columns = new ArrayList<>(); + columns.addAll(left.getTable().getColumns()); + columns.addAll(right.getTable().getColumns()); + PostgresExpressionGenerator joinGen = new PostgresExpressionGenerator(globalState).setColumns(columns); + joinExpressions.add(PostgresJoin.createJoin(left, right, PostgresJoinType.getRandom(), + joinGen.generateExpression(0, PostgresDataType.BOOLEAN))); + } + return joinExpressions; + } + + @Override + public void setOnClause(PostgresExpression clause) { + this.onClause = clause; + } + + public void setType(PostgresJoinType type) { + this.type = type; + } + + public PostgresExpression getTableReference() { + return tableReference; + } + + public PostgresExpression getLeftTable() { + return leftTable; } - public PostgresTable getTable() { - return table; + public PostgresExpression getRightTable() { + return rightTable; } public PostgresExpression getOnClause() { diff --git a/src/sqlancer/postgres/ast/PostgresLikeOperation.java b/src/sqlancer/postgres/ast/PostgresLikeOperation.java index 9504f3232..8e80e4817 100644 --- a/src/sqlancer/postgres/ast/PostgresLikeOperation.java +++ b/src/sqlancer/postgres/ast/PostgresLikeOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; import sqlancer.LikeImplementationHelper; -import sqlancer.ast.BinaryNode; +import sqlancer.common.ast.BinaryNode; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresLikeOperation extends BinaryNode implements PostgresExpression { @@ -19,6 +19,9 @@ public PostgresDataType getExpressionType() { public PostgresConstant getExpectedValue() { PostgresConstant leftVal = getLeft().getExpectedValue(); PostgresConstant rightVal = getRight().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } if (leftVal.isNull() || rightVal.isNull()) { return PostgresConstant.createNullConstant(); } else { diff --git a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java index 20f93536e..76257215b 100644 --- a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java +++ b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java @@ -5,8 +5,10 @@ public class PostgresOrderByTerm implements PostgresExpression { - private final PostgresOrder order; private final PostgresExpression expr; + private final PostgresOrder order; + private final int limit; + private final boolean ties; public enum PostgresOrder { ASC, DESC; @@ -17,18 +19,45 @@ public static PostgresOrder getRandomOrder() { } public PostgresOrderByTerm(PostgresExpression expr, PostgresOrder order) { + if (expr == null) { + throw new IllegalArgumentException("Expression cannot be null"); + } this.expr = expr; this.order = order; + + if (Randomly.getBooleanWithRatherLowProbability()) { + this.limit = (int) Randomly.getPositiveOrZeroNonCachedInteger(); + this.ties = true; + } else { + this.limit = 0; + this.ties = false; + } + } - public PostgresOrder getOrder() { - return order; + // Constructor for window functions, might be removed in the future to have only one constructor + public PostgresOrderByTerm(PostgresExpression expr, boolean ascending) { + if (expr == null) { + throw new IllegalArgumentException("Expression cannot be null"); + } + this.expr = expr; + this.order = ascending ? PostgresOrder.ASC : PostgresOrder.DESC; + this.limit = 0; + this.ties = false; } public PostgresExpression getExpr() { return expr; } + public PostgresOrder getOrder() { + return order; + } + + public boolean isAscending() { + return order == PostgresOrder.ASC; + } + @Override public PostgresConstant getExpectedValue() { throw new AssertionError(this); @@ -39,4 +68,12 @@ public PostgresDataType getExpressionType() { return null; } + @Override + public String toString() { + if (ties) { + return String.format("%s %s FETCH FIRST %d WITH TIES", expr, order, limit); + } else { + return String.format("%s %s", expr, order); + } + } } diff --git a/src/sqlancer/postgres/ast/PostgresPOSIXRegularExpression.java b/src/sqlancer/postgres/ast/PostgresPOSIXRegularExpression.java index a1e4a0362..b0affaf05 100644 --- a/src/sqlancer/postgres/ast/PostgresPOSIXRegularExpression.java +++ b/src/sqlancer/postgres/ast/PostgresPOSIXRegularExpression.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresPOSIXRegularExpression implements PostgresExpression { diff --git a/src/sqlancer/postgres/ast/PostgresPostfixOperation.java b/src/sqlancer/postgres/ast/PostgresPostfixOperation.java index 779945348..78c735030 100644 --- a/src/sqlancer/postgres/ast/PostgresPostfixOperation.java +++ b/src/sqlancer/postgres/ast/PostgresPostfixOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresPostfixOperation implements PostgresExpression { @@ -129,7 +129,11 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - return op.apply(expr.getExpectedValue()); + PostgresConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.apply(expectedValue); } public String getOperatorTextRepresentation() { diff --git a/src/sqlancer/postgres/ast/PostgresPrefixOperation.java b/src/sqlancer/postgres/ast/PostgresPrefixOperation.java index 6e875b635..8a19f7ac9 100644 --- a/src/sqlancer/postgres/ast/PostgresPrefixOperation.java +++ b/src/sqlancer/postgres/ast/PostgresPrefixOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; import sqlancer.IgnoreMeException; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresPrefixOperation implements PostgresExpression { @@ -51,7 +51,14 @@ protected PostgresConstant getExpectedValue(PostgresConstant expectedValue) { // TODO throw new IgnoreMeException(); } - return PostgresConstant.createIntConstant(-expectedValue.asInt()); + if (expectedValue.isInt() && expectedValue.asInt() == Long.MIN_VALUE) { + throw new IgnoreMeException(); + } + try { + return PostgresConstant.createIntConstant(-expectedValue.asInt()); + } catch (UnsupportedOperationException e) { + return null; + } } }; @@ -90,7 +97,11 @@ public PostgresDataType getExpressionType() { @Override public PostgresConstant getExpectedValue() { - return op.getExpectedValue(expr.getExpectedValue()); + PostgresConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.getExpectedValue(expectedValue); } public PostgresDataType[] getInputDataTypes() { diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index c1e6fa1fd..1d7172e3e 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -1,19 +1,29 @@ package sqlancer.postgres.ast; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import sqlancer.Randomly; -import sqlancer.ast.SelectBase; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; -public class PostgresSelect extends SelectBase implements PostgresExpression { +public class PostgresSelect extends SelectBase + implements PostgresExpression, Select { private SelectType selectOption = SelectType.ALL; private List joinClauses = Collections.emptyList(); private PostgresExpression distinctOnClause; private ForClause forClause; + private List windowFunctions = new ArrayList<>(); + private final Map windowDefinitions = new HashMap<>(); public enum ForClause { UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); @@ -33,6 +43,53 @@ public static ForClause getRandom() { } } + public static class WindowDefinition { + private final List partitionBy; + private final List orderBy; + private final WindowFrame frame; + + public WindowDefinition(List partitionBy, List orderBy, + WindowFrame frame) { + this.partitionBy = partitionBy; + this.orderBy = orderBy; + this.frame = frame; + } + + public List getPartitionBy() { + return partitionBy; + } + + public List getOrderBy() { + return orderBy; + } + + public WindowFrame getFrame() { + return frame; + } + } + + // Getters setters for windowfunctions + public List getWindowFunctions() { + return windowFunctions; + } + + public void setWindowFunctions(List windowFunctions) { + this.windowFunctions = windowFunctions; + } + + // Add methods for window definitions + public void addWindowDefinition(String name, WindowDefinition definition) { + windowDefinitions.put(name, definition); + } + + public WindowDefinition getWindowDefinition(String name) { + return windowDefinitions.get(name); + } + + public Map getWindowDefinitions() { + return windowDefinitions; + } + public static class PostgresFromTable implements PostgresExpression { private final PostgresTable t; private final boolean only; @@ -56,6 +113,29 @@ public PostgresDataType getExpressionType() { } } + public static class PostgresSubquery implements PostgresExpression { + private final PostgresSelect s; + private final String name; + + public PostgresSubquery(PostgresSelect s, String name) { + this.s = s; + this.name = name; + } + + public PostgresSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public PostgresDataType getExpressionType() { + return null; + } + } + public enum SelectType { DISTINCT, ALL; @@ -88,11 +168,13 @@ public PostgresDataType getExpressionType() { return null; } + @Override public void setJoinClauses(List joinStatements) { this.joinClauses = joinStatements; } + @Override public List getJoinClauses() { return joinClauses; } @@ -109,4 +191,8 @@ public ForClause getForClause() { return forClause; } + @Override + public String asString() { + return PostgresVisitor.asString(this); + } } diff --git a/src/sqlancer/postgres/ast/PostgresTableReference.java b/src/sqlancer/postgres/ast/PostgresTableReference.java new file mode 100644 index 000000000..2abf8a7d2 --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresTableReference.java @@ -0,0 +1,15 @@ +package sqlancer.postgres.ast; + +import sqlancer.postgres.PostgresSchema.PostgresTable; + +public class PostgresTableReference implements PostgresExpression { + private final PostgresTable table; + + public PostgresTableReference(PostgresTable table) { + this.table = table; + } + + public PostgresTable getTable() { + return table; + } +} diff --git a/src/sqlancer/postgres/ast/PostgresWindowFunction.java b/src/sqlancer/postgres/ast/PostgresWindowFunction.java new file mode 100644 index 000000000..15f87364f --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresWindowFunction.java @@ -0,0 +1,101 @@ +package sqlancer.postgres.ast; + +import java.util.List; + +import sqlancer.postgres.PostgresSchema.PostgresDataType; + +public class PostgresWindowFunction implements PostgresExpression { + + private final String functionName; + private final List arguments; + private final WindowSpecification windowSpec; + private final PostgresDataType returnType; + + public PostgresWindowFunction(String functionName, List arguments, + WindowSpecification windowSpec, PostgresDataType returnType) { + this.functionName = functionName; + this.arguments = arguments; + this.windowSpec = windowSpec; + this.returnType = returnType; + } + + public String getFunctionName() { + return functionName; + } + + public List getArguments() { + return arguments; + } + + public WindowSpecification getWindowSpec() { + return windowSpec; + } + + @Override + public PostgresDataType getExpressionType() { + return returnType; + } + + public static class WindowSpecification { + private final List partitionBy; + private final List orderBy; + private final WindowFrame frame; + + public WindowSpecification(List partitionBy, List orderBy, + WindowFrame frame) { + this.partitionBy = partitionBy; + this.orderBy = orderBy; + this.frame = frame; + } + + public List getPartitionBy() { + return partitionBy; + } + + public List getOrderBy() { + return orderBy; + } + + public WindowFrame getFrame() { + return frame; + } + } + + public static class WindowFrame { + public enum FrameType { + ROWS("ROWS"), RANGE("RANGE"); + + private final String sql; + + FrameType(String sql) { + this.sql = sql; + } + + public String getSQL() { + return sql; + } + } + + private final FrameType type; + private final PostgresExpression startExpr; + private final PostgresExpression endExpr; + + public WindowFrame(FrameType type, PostgresExpression startExpr, PostgresExpression endExpr) { + this.type = type; + this.startExpr = startExpr; + this.endExpr = endExpr; + } + + public FrameType getType() { + return type; + } + + public PostgresExpression getStartExpr() { + return startExpr; + } + + public PostgresExpression getEndExpr() { + return endExpr; + } + } +} diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 5d19edaa3..69b509f60 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -1,13 +1,11 @@ package sqlancer.postgres.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -23,7 +21,7 @@ public class PostgresAlterTableGenerator { private List opClasses; private PostgresGlobalState globalState; - private enum Action { + protected enum Action { // ALTER_TABLE_ADD_COLUMN, // [ COLUMN ] column data_type [ COLLATE collation ] [ // column_constraint [ ... ] ] ALTER_TABLE_DROP_COLUMN, // DROP [ COLUMN ] [ IF EXISTS ] column [ RESTRICT | CASCADE ] @@ -36,6 +34,7 @@ private enum Action { ALTER_COLUMN_SET_ATTRIBUTE_OPTION, // ALTER [ COLUMN ] column SET ( attribute_option = value [, ... ] ) ALTER_COLUMN_RESET_ATTRIBUTE_OPTION, // ALTER [ COLUMN ] column RESET ( attribute_option [, ... ] ) ALTER_COLUMN_SET_STORAGE, // ALTER [ COLUMN ] column SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } + ALTER_COLUMN_DROP_EXPRESSION, // ALTER [ COLUMN ] column DROP EXPRESSION [ IF EXISTS ] ADD_TABLE_CONSTRAINT, // ADD table_constraint [ NOT VALID ] ADD_TABLE_CONSTRAINT_USING_INDEX, // ADD table_constraint_using_index VALIDATE_CONSTRAINT, // VALIDATE CONSTRAINT constraint_name @@ -50,9 +49,12 @@ private enum Action { SET_LOGGED_UNLOGGED, // NOT_OF, // OWNER_TO, // - REPLICA_IDENTITY + REPLICA_IDENTITY, // RENAME COLUMN old_name TO new_name (for views) + ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) } + private static final List VIEW_ACTIONS = List.of(Action.ALTER_VIEW_RENAME_COLUMN); + public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { this.randomTable = randomTable; @@ -62,7 +64,8 @@ public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalStat this.opClasses = globalState.getOpClasses(); } - public static Query create(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { + public static SQLQueryAdapter create(PostgresTable randomTable, PostgresGlobalState globalState, + boolean generateOnlyKnown) { return new PostgresAlterTableGenerator(randomTable, globalState, generateOnlyKnown).generate(); } @@ -76,8 +79,7 @@ private enum Attribute { } }; - public Query generate() { - Set errors = new HashSet<>(); + public List getActions(ExpectedErrors errors) { PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonInsertUpdateErrors(errors); PostgresCommon.addCommonTableErrors(errors); @@ -93,16 +95,6 @@ public Query generate() { errors.add("could not find cast from"); errors.add("does not exist"); // TODO: investigate errors.add("constraints on permanent tables may reference only permanent tables"); - StringBuilder sb = new StringBuilder(); - sb.append("ALTER TABLE "); - if (Randomly.getBoolean()) { - sb.append(" ONLY"); - errors.add("cannot use ONLY for foreign key on partitioned table"); - } - sb.append(" "); - sb.append(randomTable.getName()); - sb.append(" "); - int i = 0; List action; if (Randomly.getBoolean()) { action = Randomly.nonEmptySubset(Action.values()); @@ -110,6 +102,20 @@ public Query generate() { // make it more likely that the ALTER TABLE succeeds action = Randomly.subset(Randomly.smallNumber(), Action.values()); } + + // If this is a view, only allow view-compatible operations + if (randomTable.isView()) { + // Remove all non-view operations + action.removeIf(a -> !VIEW_ACTIONS.contains(a)); + // If no view operations remain, add a random view operation + if (action.isEmpty()) { + action.add(VIEW_ACTIONS.get(r.getInteger(0, VIEW_ACTIONS.size() - 1))); + } + } else { + // Remove view-specific actions if this is a table + action.removeIf(VIEW_ACTIONS::contains); + } + if (randomTable.getColumns().size() == 1) { action.remove(Action.ALTER_TABLE_DROP_COLUMN); } @@ -118,9 +124,41 @@ public Query generate() { action.remove(Action.CLUSTER_ON); } action.remove(Action.SET_WITH_OIDS); + if (!randomTable.hasIndexes()) { + action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); + } + if (randomTable.isPartitioned()) { + action.remove(Action.SET_LOGGED_UNLOGGED); + } if (action.isEmpty()) { throw new IgnoreMeException(); } + return action; + } + + public SQLQueryAdapter generate() { + ExpectedErrors errors = new ExpectedErrors(); + int i = 0; + List action = getActions(errors); + StringBuilder sb = new StringBuilder(); + + // Check if we're dealing with a view operation + boolean isViewOperation = action.contains(Action.ALTER_VIEW_RENAME_COLUMN); + + if (isViewOperation) { + sb.append("ALTER VIEW "); + } else { + sb.append("ALTER TABLE "); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + errors.add("cannot use ONLY for foreign key on partitioned table"); + } + } + + sb.append(" "); + sb.append(randomTable.getName()); + sb.append(" "); + for (Action a : action) { if (i++ != 0) { sb.append(", "); @@ -199,6 +237,9 @@ public Query generate() { sb.append("DROP NOT NULL"); errors.add("is in a primary key"); errors.add("is an identity column"); + errors.add("is in index used as replica identity"); + // PG18 update: otherwise we need to encode contraint inheritance info in PostgreColumn + errors.add("cannot drop inherited constraint"); } break; case ALTER_COLUMN_SET_STATISTICS: @@ -242,12 +283,28 @@ public Query generate() { errors.add("can only have storage"); errors.add("is an identity column"); break; + case ALTER_COLUMN_DROP_EXPRESSION: + alterColumn(randomTable, sb); + sb.append("DROP EXPRESSION"); + if (Randomly.getBoolean()) { + sb.append(" IF EXISTS"); + } + errors.add("is not a generated column"); + errors.add("is not a stored generated column"); + errors.add("cannot drop expression from inherited column"); + errors.add("cannot drop generation expression from inherited column"); + errors.add("must be applied to child tables too"); + errors.add("cannot drop expression from column"); + break; case ADD_TABLE_CONSTRAINT: sb.append("ADD "); + sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); PostgresCommon.addTableConstraint(sb, randomTable, globalState, errors); + errors.add("already exists"); errors.add("multiple primary keys for table"); errors.add("could not create unique index"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("cannot cast type"); errors.add("unsupported PRIMARY KEY constraint with partition key definition"); errors.add("unsupported UNIQUE constraint with partition key definition"); @@ -273,8 +330,9 @@ public Query generate() { break; case ADD_TABLE_CONSTRAINT_USING_INDEX: sb.append("ADD "); - // sb.append("CONSTRAINT 'asdf' "); + sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); + errors.add("already exists"); errors.add("not valid"); sb.append(" USING INDEX "); sb.append(randomTable.getRandomIndex().getIndexName()); @@ -285,6 +343,7 @@ public Query generate() { errors.add("appears twice in unique constraint"); errors.add("appears twice in primary key constraint"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("insufficient columns in PRIMARY KEY constraint definition"); errors.add("which is part of the partition key"); break; @@ -344,7 +403,8 @@ public Query generate() { if (Randomly.getBoolean() || randomTable.getIndexes().isEmpty()) { sb.append(Randomly.fromOptions("DEFAULT", "FULL", "NOTHING")); } else { - sb.append("USING INDEX " + randomTable.getRandomIndex().getIndexName()); + sb.append("USING INDEX "); + sb.append(randomTable.getRandomIndex().getIndexName()); errors.add("cannot be used as replica identity"); errors.add("cannot use non-unique index"); errors.add("cannot use expression index"); @@ -352,12 +412,23 @@ public Query generate() { errors.add("cannot use invalid index"); } break; + case ALTER_VIEW_RENAME_COLUMN: + sb.append("RENAME COLUMN "); + PostgresColumn columnToRename = randomTable.getRandomColumn(); + sb.append(columnToRename.getName()); + sb.append(" TO "); + sb.append("new_" + columnToRename.getName() + "_" + r.getInteger(1, 1000)); + errors.add("column does not exist"); + errors.add("column name already exists"); + errors.add("cannot rename column of view"); + errors.add("permission denied"); + break; default: throw new AssertionError(a); } } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } private static void alterColumn(PostgresTable randomTable, StringBuilder sb) { diff --git a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java index 7a4a64427..3acbfde10 100644 --- a/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAnalyzeGenerator.java @@ -1,11 +1,10 @@ package sqlancer.postgres.gen; -import java.util.Arrays; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -14,7 +13,7 @@ public final class PostgresAnalyzeGenerator { private PostgresAnalyzeGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(); StringBuilder sb = new StringBuilder("ANALYZE"); if (Randomly.getBoolean()) { @@ -37,7 +36,7 @@ public static Query create(PostgresGlobalState globalState) { } } // FIXME: bug in postgres? - return new QueryAdapter(sb.toString(), Arrays.asList("deadlock")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("deadlock")); } } diff --git a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java index 00f364bdc..9d966e8e5 100644 --- a/src/sqlancer/postgres/gen/PostgresClusterGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresClusterGenerator.java @@ -1,11 +1,8 @@ package sqlancer.postgres.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -14,8 +11,8 @@ public final class PostgresClusterGenerator { private PostgresClusterGenerator() { } - public static Query create(PostgresGlobalState globalState) { - Set errors = new HashSet<>(); + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); errors.add("there is no previously clustered index for table"); errors.add("cannot cluster a partitioned table"); errors.add("access method does not support clustering"); @@ -29,7 +26,7 @@ public static Query create(PostgresGlobalState globalState) { errors.add("cannot cluster on partial index"); } } - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java index df668c372..adcce3d82 100644 --- a/src/sqlancer/postgres/gen/PostgresCommentGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresCommentGenerator.java @@ -1,14 +1,13 @@ package sqlancer.postgres.gen; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; /** - * @see https://www.postgresql.org/docs/devel/sql-comment.html + * @see COMMENT */ public final class PostgresCommentGenerator { @@ -19,7 +18,7 @@ private enum Action { INDEX, COLUMN, STATISTICS, TABLE } - public static Query generate(PostgresGlobalState globalState) { + public static SQLQueryAdapter generate(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("COMMENT ON "); Action type = Randomly.fromOptions(Action.values()); @@ -63,7 +62,7 @@ public static Query generate(PostgresGlobalState globalState) { sb.append(globalState.getRandomly().getString().replace("'", "''")); sb.append("'"); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index f45180467..180b4e449 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -3,13 +3,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -22,20 +23,45 @@ public final class PostgresCommon { private PostgresCommon() { } - public static void addCommonFetchErrors(Set errors) { + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); errors.add("but it cannot be referenced from this part of the query"); errors.add("missing FROM-clause entry for table"); errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in GROUP BY"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + + return errors; } - public static void addCommonTableErrors(Set errors) { + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude + errors.add("partitioned tables cannot be unlogged"); + + return errors; } - public static void addCommonExpressionErrors(Set errors) { + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("for encoding \"SQL_ASCII\" does not exist"); + errors.add("invalid byte sequence for encoding"); errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); @@ -49,7 +75,6 @@ public static void addCommonExpressionErrors(Set errors) { errors.add("invalid hexadecimal digit"); errors.add("invalid hexadecimal data: odd number of digits"); errors.add("zero raised to a negative power is undefined"); - errors.add("cannot convert infinity to numeric"); errors.add("division by zero"); errors.add("invalid input syntax for type money"); errors.add("invalid input syntax for type"); @@ -59,14 +84,33 @@ public static void addCommonExpressionErrors(Set errors) { errors.add("is of type boolean but expression is of type text"); errors.add("a negative number raised to a non-integer power yields a complex result"); errors.add("could not determine polymorphic type because input has type unknown"); - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); + errors.add("character number must be positive"); + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; } - private static void addToCharFunctionErrors(Set errors) { + public static List getCommonExpressionRegexErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add(Pattern.compile("cannot convert infinity to \\w+")); + errors.addAll(getFunctionRegexErrors()); + + return errors; + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + errors.addAllRegexes(getCommonExpressionRegexErrors()); + } + + private static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("multiple decimal points"); errors.add("and decimal point together"); errors.add("multiple decimal points"); @@ -78,16 +122,26 @@ private static void addToCharFunctionErrors(Set errors) { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + errors.add("\"EEEE\" must be the last pattern used"); + errors.add("is incompatible with other formats"); + + return errors; } - private static void addBitStringOperationErrors(Set errors) { + private static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("cannot XOR bit strings of different sizes"); errors.add("cannot AND bit strings of different sizes"); errors.add("cannot OR bit strings of different sizes"); errors.add("must be type boolean, not type text"); + + return errors; } - private static void addFunctionErrors(Set errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("out of valid range"); // get_bit/get_byte errors.add("cannot take logarithm of a negative number"); errors.add("cannot take logarithm of zero"); @@ -96,25 +150,82 @@ private static void addFunctionErrors(Set errors) { errors.add("requested character not valid for encoding"); // chr errors.add("requested length too large"); // repeat errors.add("invalid memory alloc request size"); // repeat - errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr errors.add("invalid mask length"); // set_masklen + + return errors; + } + + private static List getFunctionRegexErrors() { + ArrayList errors = new ArrayList<>(); + /* + * PostgreSQL support only a few conversion variants to ASCII: LATIN1, LATIN2, LATIN9 and WINDOWS1250. So, it is + * better to skip this error at all. + */ + errors.add(Pattern.compile("encoding conversion from \\w+ to ASCII not supported")); + + /* + * In accordance with PostgreSQL code, commit 0ab1a2e, conversions to or from SQL_ASCII is meaningless. So + * disable errors on such an attempt. + */ + errors.add(Pattern.compile("encoding conversion from SQL_ASCII to \\w+ not supported")); + errors.add(Pattern.compile("encoding conversion from \\w+ to SQL_ASCII not supported")); + + return errors; } - private static void addCommonRegexExpressionErrors(Set errors) { + private static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not a valid hexadecimal digit"); + + return errors; } - public static void addCommonRangeExpressionErrors(Set errors) { + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("range lower bound must be less than or equal to range upper bound"); errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + + return errors; } - public static void addCommonInsertUpdateErrors(Set errors) { + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); + } + + public static List getCommonInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("value too long for type character"); + errors.add("cannot insert a non-DEFAULT value into column"); errors.add("not found in view targetlist"); + + return errors; + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getCommonInsertUpdateErrors()); + } + + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in GROUP BY"); + + return errors; + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); } public static boolean appendDataType(PostgresDataType type, StringBuilder sb, boolean allowSerial, @@ -147,7 +258,7 @@ public static boolean appendDataType(PostgresDataType type, StringBuilder sb, bo } else { sb.append("name"); } - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() && !PostgresProvider.generateOnlyKnown) { sb.append(" COLLATE "); sb.append('"'); sb.append(Randomly.fromList(opClasses)); @@ -219,7 +330,7 @@ private enum StorageParameters { } } - public static void generateWith(StringBuilder sb, PostgresGlobalState globalState, Set errors) { + public static void generateWith(StringBuilder sb, PostgresGlobalState globalState, ExpectedErrors errors) { if (Randomly.getBoolean()) { sb.append(" WITH ("); ArrayList values = new ArrayList<>(Arrays.asList(StorageParameters.values())); @@ -241,7 +352,7 @@ public static void generateWith(StringBuilder sb, PostgresGlobalState globalStat } public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder sb, PostgresTable table, - PostgresGlobalState globalState, Set errors) { + PostgresGlobalState globalState, ExpectedErrors errors) { // TODO constraint name List tableConstraints = Randomly.nonEmptySubset(TableConstraints.values()); if (excludePrimaryKey) { @@ -258,12 +369,12 @@ public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder } public static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, - Set errors) { + ExpectedErrors errors) { addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); } private static void addTableConstraint(StringBuilder sb, PostgresTable table, PostgresGlobalState globalState, - TableConstraints t, Set errors) { + TableConstraints t, ExpectedErrors errors) { List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); List otherColumns; PostgresCommon.addCommonExpressionErrors(errors); @@ -328,6 +439,8 @@ private static void addTableConstraint(StringBuilder sb, PostgresTable table, Po } break; case EXCLUDE: + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("unsupported EXCLUDE constraint with partition key definition"); sb.append("EXCLUDE "); sb.append("("); // TODO [USING index_method ] @@ -348,7 +461,6 @@ private static void addTableConstraint(StringBuilder sb, PostgresTable table, Po errors.add("exclusion constraints are not supported on partitioned tables"); errors.add("The exclusion operator must be related to the index operator class for the constraint"); errors.add("could not create exclusion constraint"); - // TODO: index parameters if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append("("); @@ -362,7 +474,8 @@ private static void addTableConstraint(StringBuilder sb, PostgresTable table, Po } } - private static void appendIndexParameters(StringBuilder sb, PostgresGlobalState globalState, Set errors) { + private static void appendIndexParameters(StringBuilder sb, PostgresGlobalState globalState, + ExpectedErrors errors) { if (Randomly.getBoolean()) { generateWith(sb, globalState, errors); } @@ -402,12 +515,4 @@ private static void appendExcludeElement(StringBuilder sb, PostgresGlobalState g private static void deleteOrUpdateAction(StringBuilder sb) { sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); } - - public static void addGroupingErrors(Set errors) { - errors.add("non-integer constant in GROUP BY"); // TODO - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("is not in select list"); - errors.add("aggregate functions are not allowed in GROUP BY"); - } - } diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 1fc592235..3250b86c8 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -1,41 +1,38 @@ package sqlancer.postgres.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; -public final class PostgresDeleteGenerator { +public final class PostgresDeleteGenerator extends AbstractDeleteGenerator { + + private final PostgresGlobalState globalState; + + private PostgresDeleteGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; + } - private PostgresDeleteGenerator() { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + return new PostgresDeleteGenerator(globalState).getStatement(); } - public static Query create(PostgresGlobalState globalState) { + @Override + public void buildStatement() { PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - Set errors = new HashSet<>(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); - if (Randomly.getBoolean()) { - sb.append(" ONLY"); - } - sb.append(" "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName(), Randomly.getBoolean()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(PostgresVisitor.asString(PostgresExpressionGenerator.generateExpression(globalState, + appendWhereClause(PostgresVisitor.asString(PostgresExpressionGenerator.generateExpression(globalState, table.getColumns(), PostgresDataType.BOOLEAN))); } if (Randomly.getBoolean()) { - sb.append(" RETURNING "); - sb.append(PostgresVisitor + appendReturningClause(PostgresVisitor .asString(PostgresExpressionGenerator.generateExpression(globalState, table.getColumns()))); } PostgresCommon.addCommonExpressionErrors(errors); @@ -43,7 +40,6 @@ public static Query create(PostgresGlobalState globalState) { errors.add("cannot cast"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new QueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java index 4dce13a48..0c02efcd0 100644 --- a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java @@ -1,10 +1,8 @@ package sqlancer.postgres.gen; -import java.util.Arrays; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable.TableType; @@ -13,7 +11,7 @@ public final class PostgresDiscardGenerator { private PostgresDiscardGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("DISCARD "); // prevent that DISCARD discards all tables (if they are TEMP tables) @@ -26,7 +24,8 @@ public static Query create(PostgresGlobalState globalState) { what = Randomly.fromOptions("PLANS", "SEQUENCES"); } sb.append(what); - return new QueryAdapter(sb.toString(), Arrays.asList("cannot run inside a transaction block")) { + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + private static final long serialVersionUID = 1L; @Override public boolean couldAffectSchema() { diff --git a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java index 79930f6cf..4ed133f0b 100644 --- a/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDropIndexGenerator.java @@ -1,21 +1,20 @@ package sqlancer.postgres.gen; -import java.util.Arrays; import java.util.List; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; -import sqlancer.sqlite3.gen.SQLite3Common; public final class PostgresDropIndexGenerator { private PostgresDropIndexGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { List indexes = globalState.getSchema().getRandomTable().getIndexes(); StringBuilder sb = new StringBuilder(); sb.append("DROP INDEX "); @@ -26,7 +25,7 @@ public static Query create(PostgresGlobalState globalState) { sb.append(", "); } if (indexes.isEmpty() || Randomly.getBoolean()) { - sb.append(SQLite3Common.createIndexName(Randomly.smallNumber())); + sb.append(DBMSCommon.createIndexName(Randomly.smallNumber())); } else { sb.append(Randomly.fromList(indexes).getIndexName()); } @@ -43,9 +42,9 @@ public static Query create(PostgresGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new QueryAdapter(sb.toString(), - Arrays.asList("cannot drop desired object(s) because other objects depend on them", "cannot drop index", - "does not exist"), + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", + "cannot drop index", "does not exist"), true); } diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java new file mode 100644 index 000000000..e4359e5aa --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -0,0 +1,82 @@ +package sqlancer.postgres.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.ast.PostgresSelect; + +public final class PostgresExplainGenerator { + + private PostgresExplainGenerator() { + + } + + public static String explain(String selectStr) { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN (FORMAT JSON) "); + sb.append(selectStr); + return sb.toString(); + } + + public static String explainGeneral(String selectStr) { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + + List options = new ArrayList<>(); + boolean analyze = Randomly.getBoolean(); + boolean genericPlan = !analyze && Randomly.getBoolean(); + if (analyze) { + options.add("ANALYZE"); + } + if (genericPlan) { + options.add("GENERIC_PLAN"); + } + if (Randomly.getBoolean()) { + options.add("FORMAT " + Randomly.fromOptions("TEXT", "XML", "JSON", "YAML")); + } + if (Randomly.getBoolean()) { + options.add("VERBOSE"); + } + if (Randomly.getBoolean()) { + options.add("COSTS"); + } + if (analyze && Randomly.getBoolean()) { + options.add("BUFFERS"); + } + if (analyze && Randomly.getBoolean()) { + options.add("TIMING"); + } + if (Randomly.getBoolean()) { + options.add("SUMMARY"); + } + if (!options.isEmpty()) { + sb.append("("); + sb.append(String.join(", ", options)); + sb.append(") "); + } + + sb.append(selectStr); + return sb.toString(); + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) throws Exception { + PostgresSchema.PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + gen.setTablesAndColumns(new PostgresTables(Arrays.asList(table))); + PostgresSelect select = gen.generateSelect(); + select.setFromList(gen.getTableRefs()); + select.setFetchColumns(gen.generateFetchColumns(false)); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PostgresDataType.BOOLEAN)); + } + return new SQLQueryAdapter(explainGeneral(select.asString())); + } + +} diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index d5a0ec3b9..bad87affc 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -4,17 +4,26 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.postgres.PostgresBugs; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresRowValue; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresSchema.PostgresTables; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresAggregate.PostgresAggregateFunction; import sqlancer.postgres.ast.PostgresBetweenOperation; @@ -38,18 +47,32 @@ import sqlancer.postgres.ast.PostgresFunction.PostgresFunctionWithResult; import sqlancer.postgres.ast.PostgresFunctionWithUnknownResult; import sqlancer.postgres.ast.PostgresInOperation; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; -import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression.POSIXRegex; import sqlancer.postgres.ast.PostgresPostfixOperation; import sqlancer.postgres.ast.PostgresPostfixOperation.PostfixOperator; +import sqlancer.postgres.ast.PostgresPostfixText; import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresPrefixOperation.PrefixOperator; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.ast.PostgresSelect.ForClause; +import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; +import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; -public class PostgresExpressionGenerator { +public class PostgresExpressionGenerator implements ExpressionGenerator, + NoRECGenerator, + TLPWhereGenerator, + CERTGenerator { private final int maxDepth; @@ -57,6 +80,8 @@ public class PostgresExpressionGenerator { private List columns; + private List targetTables; + private PostgresRowValue rw; private boolean expectedResult; @@ -65,10 +90,16 @@ public class PostgresExpressionGenerator { private boolean allowAggregateFunctions; + private final Map functionsAndTypes; + + private final List allowedFunctionTypes; + public PostgresExpressionGenerator(PostgresGlobalState globalState) { this.r = globalState.getRandomly(); this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); this.globalState = globalState; + this.functionsAndTypes = globalState.getFunctionsAndTypes(); + this.allowedFunctionTypes = globalState.getAllowedFunctionTypes(); } public PostgresExpressionGenerator setColumns(List columns) { @@ -81,24 +112,16 @@ public PostgresExpressionGenerator setRowValue(PostgresRowValue rw) { return this; } - public PostgresExpressionGenerator expectedResult() { - this.expectedResult = true; - return this; - } - - public static PostgresExpression generateExpression(PostgresGlobalState globalState) { - return new PostgresExpressionGenerator(globalState).generateExpression(0); - } - - PostgresExpression generateExpression(int depth) { + public PostgresExpression generateExpression(int depth) { return generateExpression(depth, PostgresDataType.getRandomType()); } - public List generateOrderBy() { + @Override + public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { - orderBys.add(new PostgresOrderByTerm(PostgresColumnValue.create(Randomly.fromList(columns), null), - PostgresOrder.getRandomOrder())); + PostgresExpression expr = PostgresColumnValue.create(Randomly.fromList(columns), null); + orderBys.add(expr); } return orderBys; } @@ -111,6 +134,10 @@ private enum BooleanExpression { private PostgresExpression generateFunctionWithUnknownResult(int depth, PostgresDataType type) { List supportedFunctions = PostgresFunctionWithUnknownResult .getSupportedFunctions(type); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + supportedFunctions = supportedFunctions.stream() + .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); if (supportedFunctions.isEmpty()) { throw new IgnoreMeException(); } @@ -121,6 +148,9 @@ private PostgresExpression generateFunctionWithUnknownResult(int depth, Postgres private PostgresExpression generateFunctionWithKnownResult(int depth, PostgresDataType type) { List functions = Stream.of(PostgresFunction.PostgresFunctionWithResult.values()) .filter(f -> f.supportsReturnType(type)).collect(Collectors.toList()); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); if (functions.isEmpty()) { throw new IgnoreMeException(); } @@ -200,8 +230,7 @@ private PostgresExpression generateBooleanExpression(int depth) { } private PostgresDataType getMeaningfulType() { - // make it more likely that the expression does not only consist of constant - // expressions + // make it more likely that the expression does not only consist of constant expressions if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { return PostgresDataType.getRandomType(); } else { @@ -256,8 +285,21 @@ public PostgresExpression generateExpression(int depth, PostgresDataType origina dataType = PostgresDataType.INT; } if (!filterColumns(dataType).isEmpty() && Randomly.getBoolean()) { - return createColumnOfType(dataType); + return potentiallyWrapInCollate(dataType, createColumnOfType(dataType)); } + PostgresExpression exprInternal = generateExpressionInternal(depth, dataType); + return potentiallyWrapInCollate(dataType, exprInternal); + } + + private PostgresExpression potentiallyWrapInCollate(PostgresDataType dataType, PostgresExpression exprInternal) { + if (dataType == PostgresDataType.TEXT && PostgresProvider.generateOnlyKnown) { + return new PostgresCollate(exprInternal, "C"); + } else { + return exprInternal; + } + } + + private PostgresExpression generateExpressionInternal(int depth, PostgresDataType dataType) throws AssertionError { if (allowAggregateFunctions && Randomly.getBoolean()) { allowAggregateFunctions = false; // aggregate function calls cannot be nested return getAggregate(dataType); @@ -318,7 +360,10 @@ private static PostgresCompoundDataType getCompoundDataType(PostgresDataType typ return PostgresCompoundDataType.create(type); case TEXT: // TODO case BIT: - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() || PostgresProvider.generateOnlyKnown /* + * The PQS implementation does not check for + * size specifications + */) { return PostgresCompoundDataType.create(type); } else { return PostgresCompoundDataType.create(type, (int) Randomly.getNotCachedInteger(1, 1000)); @@ -357,7 +402,7 @@ private PostgresExpression generateTextExpression(int depth) { if (expectedResult) { validOptions.remove(TextExpression.COLLATE); } - if (!globalState.getDmbsSpecificOptions().testCollations) { + if (!globalState.getDbmsSpecificOptions().testCollations) { validOptions.remove(TextExpression.COLLATE); } option = Randomly.fromList(validOptions); @@ -378,6 +423,66 @@ private PostgresExpression generateTextExpression(int depth) { } } + public PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { + List arguments = generateWindowFunctionArguments(depth); + List partitionBy = generatePartitionByExpressions(depth); + List orderBy = generateOrderByExpressions(depth); + WindowFrame frame = generateWindowFrame(); + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + String functionName = selectWindowFunctionName(); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } + + private List generateWindowFunctionArguments(int depth) { + List arguments = new ArrayList<>(); + if (Randomly.getBoolean()) { + arguments.add(generateExpression(depth + 1)); + } + return arguments; + } + + private List generatePartitionByExpressions(int depth) { + List partitionBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + partitionBy.add(generateExpression(depth + 1)); + } + } + return partitionBy; + } + + private List generateOrderByExpressions(int depth) { + List orderBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + PostgresExpression expr = generateExpression(depth + 1); + // Call the second constructor in PostgresOrderByTerm, might be removed in the future to have only one + // constructor + orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + } + } + return orderBy; + } + + private WindowFrame generateWindowFrame() { + if (Randomly.getBoolean()) { + WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); + PostgresExpression startExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + PostgresExpression endExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + return new WindowFrame(frameType, startExpr, endExpr); + } + return null; + } + + private String selectWindowFunctionName() { + return Randomly.fromList(Arrays.asList("row_number", "rank", "dense_rank", "percent_rank", "cume_dist", "ntile", + "lag", "lead", "first_value", "last_value", "nth_value")); + } + private PostgresExpression generateConcat(int depth) { PostgresExpression left = generateExpression(depth + 1, PostgresDataType.TEXT); PostgresExpression right = generateExpression(depth + 1); @@ -401,6 +506,7 @@ private PostgresExpression generateBitExpression(int depth) { } } + // Removed WINDOW_FUNCTION option from the integer expression generation. private enum IntExpression { UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION } @@ -440,19 +546,15 @@ final List filterColumns(PostgresDataType type) { } } - public static PostgresExpression generateConstant(Randomly r) { - return generateConstant(r, Randomly.fromOptions(PostgresDataType.values())); - } - - public static PostgresExpression generateTrueCondition(List columns, PostgresRowValue rw, - PostgresGlobalState globalState) { - PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) - .expectedResult().generateExpression(0, PostgresDataType.BOOLEAN); - if (expr.getExpectedValue().isNull()) { - return PostgresPostfixOperation.create(expr, PostfixOperator.IS_NULL); - } - return PostgresPostfixOperation.create(expr, expr.getExpectedValue().cast(PostgresDataType.BOOLEAN).asBoolean() - ? PostfixOperator.IS_TRUE : PostfixOperator.IS_FALSE); + public PostgresExpression generateExpressionWithExpectedResult(PostgresDataType type) { + this.expectedResult = true; + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns) + .setRowValue(rw); + PostgresExpression expr; + do { + expr = gen.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; } public static PostgresExpression generateConstant(Randomly r, PostgresDataType type) { @@ -470,7 +572,7 @@ public static PostgresExpression generateConstant(Randomly r, PostgresDataType t return PostgresConstant.createIntConstant(r.getInteger()); } case BOOLEAN: - if (Randomly.getBooleanWithSmallProbability()) { + if (Randomly.getBooleanWithSmallProbability() && !PostgresProvider.generateOnlyKnown) { return PostgresConstant .createTextConstant(Randomly.fromOptions("TR", "TRUE", "FA", "FALSE", "0", "1", "ON", "off")); } else { @@ -515,12 +617,6 @@ public static PostgresExpression generateExpression(PostgresGlobalState globalSt return new PostgresExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); } - public static PostgresExpression generateExpression(PostgresGlobalState globalState, List columns, - PostgresDataType type, PostgresRowValue rw) { - return new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw).generateExpression(0, - type); - } - public static PostgresExpression generateExpression(PostgresGlobalState globalState, List columns) { return new PostgresExpressionGenerator(globalState).setColumns(columns).generateExpression(0); @@ -574,4 +670,306 @@ public PostgresExpressionGenerator allowAggregates(boolean value) { return this; } + public static PostgresSubquery createSubquery(PostgresGlobalState globalState, String name, PostgresTables tables) { + List columns = new ArrayList<>(); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + PostgresSelect select = new PostgresSelect(); + select.setFromList(tables.getTables().stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return new PostgresSubquery(select, name); + } + + @Override + public PostgresExpression generatePredicate() { + return generateExpression(PostgresDataType.BOOLEAN); + } + + @Override + public PostgresExpression negatePredicate(PostgresExpression predicate) { + return new PostgresPrefixOperation(predicate, PostgresPrefixOperation.PrefixOperator.NOT); + } + + @Override + public PostgresExpression isNull(PostgresExpression expr) { + return new PostgresPostfixOperation(expr, PostfixOperator.IS_NULL); + } + + @Override + public PostgresExpressionGenerator setTablesAndColumns( + sqlancer.common.schema.AbstractTables targetTables) { + this.targetTables = targetTables.getTables(); + this.columns = targetTables.getColumns(); + return this; + } + + @Override + public PostgresExpression generateBooleanExpression() { + return generateExpression(PostgresDataType.BOOLEAN); + } + + @Override + public PostgresSelect generateSelect() { + PostgresSelect select = new PostgresSelect(); + + if (Randomly.getBooleanWithRatherLowProbability()) { + List windowFunctions = generateWindowFunctions(); + select.setWindowFunctions(windowFunctions); + } + + return select; + } + + private List generateWindowFunctions() { + List windowFunctions = new ArrayList<>(); + int numWindowFunctions = Randomly.smallNumber(); + for (int i = 0; i < numWindowFunctions; i++) { + windowFunctions.add(generateWindowFunction(0, + Randomly.fromList(Arrays.asList(PostgresDataType.INT, PostgresDataType.FLOAT)))); + } + return windowFunctions; + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + for (int i = 1; i < targetTables.size(); i++) { + PostgresExpression joinClause = generateExpression(PostgresDataType.BOOLEAN); + PostgresTable table = Randomly.fromList(targetTables); + targetTables.remove(table); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PostgresSubquery subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); + PostgresExpression joinClause = generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return targetTables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new PostgresColumnValue(PostgresColumn.createDummy("*"), null)); + } + allowAggregateFunctions = true; + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(columns); + for (PostgresColumn c : targetColumns) { + fetchColumns.add(new PostgresColumnValue(c, null)); + } + allowAggregateFunctions = false; + return fetchColumns; + } + + @Override + public String generateOptimizedQueryString(PostgresSelect select, PostgresExpression whereCondition, + boolean shouldUseAggregate) { + PostgresColumnValue allColumns = new PostgresColumnValue(PostgresColumn.createDummy("*"), null); + if (shouldUseAggregate) { + select.setFetchColumns( + Arrays.asList(new PostgresAggregate(List.of(allColumns), PostgresAggregateFunction.COUNT))); + } else { + select.setFetchColumns(Arrays.asList(allColumns)); + } + select.setWhereClause(whereCondition); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + select.setSelectType(SelectType.ALL); + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpression whereCondition) { + PostgresCastOperation isTrue = new PostgresCastOperation(whereCondition, + PostgresCompoundDataType.create(PostgresDataType.INT)); + PostgresPostfixText asText = new PostgresPostfixText(isTrue, " as count", null, PostgresDataType.INT); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + select.setOrderByClauses(List.of()); + select.setSelectType(SelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + + @Override + public String generateExplainQuery(PostgresSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(PostgresSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateJoin); + mutators.add(this::mutateWhere); + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateWindowFunction); + if (!PostgresBugs.bug18643) { + mutators.add(this::mutateAnd); + mutators.add(this::mutateOr); + } + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + private boolean mutateWindowFunction(PostgresSelect select) { + List windowFunctions = select.getWindowFunctions(); + if (windowFunctions == null || windowFunctions.isEmpty()) { + windowFunctions = new ArrayList<>(); + windowFunctions.add(generateWindowFunction(0, PostgresDataType.INT)); + select.setWindowFunctions(windowFunctions); + return false; + } else { + windowFunctions.remove(Randomly.fromList(windowFunctions)); + if (windowFunctions.isEmpty()) { + select.setWindowFunctions(null); + } + return true; + } + } + + boolean mutateJoin(PostgresSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + PostgresJoin join = (PostgresJoin) Randomly.fromList(select.getJoinList()); + + // Exclude CROSS for on condition + if (join.getType() == PostgresJoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((PostgresTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((PostgresTableReference) join.getRightTable()).getTable().getColumns()); + PostgresExpressionGenerator joinGen2 = new PostgresExpressionGenerator(globalState).setColumns(columns); + join.setOnClause(joinGen2.generateExpression(0, PostgresDataType.BOOLEAN)); + } + + PostgresJoinType newJoinType = PostgresJoinType.INNER; + if (join.getType() == PostgresJoinType.LEFT || join.getType() == PostgresJoinType.RIGHT) { + newJoinType = PostgresJoinType.getRandomExcept(PostgresJoinType.LEFT, PostgresJoinType.RIGHT); + } else { + newJoinType = PostgresJoinType.getRandomExcept(join.getType()); + } + boolean increase = join.getType().ordinal() < newJoinType.ordinal(); + join.setType(newJoinType); + if (newJoinType == PostgresJoinType.CROSS) { + join.setOnClause(null); + } + return increase; + } + + boolean mutateDistinct(PostgresSelect select) { + PostgresSelect.SelectType selectType = select.getSelectOption(); + if (selectType != PostgresSelect.SelectType.ALL) { + select.setSelectType(PostgresSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(PostgresSelect.SelectType.DISTINCT); + return false; + } + } + + boolean mutateWhere(PostgresSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + } + return increase; + } + + boolean mutateGroupBy(PostgresSelect select) { + boolean increase = !select.getGroupByExpressions().isEmpty(); + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(PostgresSelect select) { + if (select.getGroupByExpressions().isEmpty()) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(PostgresSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(PostgresSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(PostgresSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + Randomly r = new Randomly(); + select.setLimitClause(PostgresConstant.createIntConstant((int) Math.abs(r.getInteger()))); + } + return increase; + } } diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 410a970f7..c852684a9 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -1,13 +1,12 @@ package sqlancer.postgres.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -15,25 +14,26 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.sqlite3.gen.SQLite3Common; -public final class PostgresIndexGenerator { +public class PostgresIndexGenerator extends AbstractIndexGenerator { - private PostgresIndexGenerator() { + private final PostgresGlobalState globalState; + + public PostgresIndexGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; } public enum IndexType { BTREE, HASH, GIST, GIN } - public static Query generate(PostgresGlobalState globalState) { - Set errors = new HashSet<>(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - if (Randomly.getBoolean()) { - sb.append(" UNIQUE"); - } - sb.append(" INDEX "); + public static SQLQueryAdapter generate(PostgresGlobalState globalState) { + return new PostgresIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(Randomly.getBoolean()); /* * Commented out as a workaround for https://www.postgresql.org/message-id/CA%2Bu7OA4XYhc- * qyCgJqwwgMGZDWAyeH821oa5oMzm_HEifZ4BeA%40mail.gmail.com @@ -107,10 +107,9 @@ public static Query generate(PostgresGlobalState globalState) { sb.append(")"); } if (Randomly.getBoolean()) { - sb.append(" WHERE "); PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(randomTable.getColumns()) .setGlobalState(globalState).generateExpression(PostgresDataType.BOOLEAN); - sb.append(PostgresVisitor.asString(expr)); + appendWhereClause(PostgresVisitor.asString(expr)); } errors.add("already contains data"); // CONCURRENT INDEX failed errors.add("You might need to add explicit type casts"); @@ -136,15 +135,15 @@ public static Query generate(PostgresGlobalState globalState) { errors.add("functions in index predicate must be marked IMMUTABLE"); errors.add("functions in index expression must be marked IMMUTABLE"); errors.add("result of range difference would not be contiguous"); + errors.add("which is part of the partition key"); PostgresCommon.addCommonExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors); } private static String getNewIndexName(PostgresTable randomTable) { List indexes = randomTable.getIndexes(); int indexI = 0; while (true) { - String indexName = SQLite3Common.createIndexName(indexI++); + String indexName = DBMSCommon.createIndexName(indexI++); if (indexes.stream().noneMatch(i -> i.getIndexName().equals(indexName))) { return indexName; } diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 759249326..49b94b184 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -1,14 +1,13 @@ package sqlancer.postgres.gen; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; @@ -19,9 +18,13 @@ public final class PostgresInsertGenerator { private PostgresInsertGenerator() { } - public static Query insert(PostgresGlobalState globalState) { + public static SQLQueryAdapter insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); - Set errors = new HashSet<>(); + return insertRows(globalState, table); + } + + public static SQLQueryAdapter insertRows(PostgresGlobalState globalState, PostgresSchema.PostgresTable table) { + ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonInsertUpdateErrors(errors); @@ -51,7 +54,7 @@ public static Query insert(PostgresGlobalState globalState) { } sb.append(" VALUES"); - if (globalState.getDmbsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { StringBuilder sbRowValue = new StringBuilder(); sbRowValue.append("("); for (int i = 0; i < columns.size(); i++) { @@ -98,7 +101,7 @@ public static Query insert(PostgresGlobalState globalState) { errors.add("division by zero"); errors.add("violates foreign key constraint"); errors.add("data type unknown"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } private static void insertRow(PostgresGlobalState globalState, StringBuilder sb, List columns, diff --git a/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java b/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java index a75b4e26a..ef044dc3d 100644 --- a/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresNotifyGenerator.java @@ -1,8 +1,7 @@ package sqlancer.postgres.gen; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresNotifyGenerator { @@ -14,7 +13,7 @@ private static String getChannel() { return Randomly.fromOptions("asdf", "test"); } - public static Query createNotify(PostgresGlobalState globalState) { + public static SQLQueryAdapter createNotify(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("NOTIFY "); sb.append(getChannel()); @@ -24,17 +23,17 @@ public static Query createNotify(PostgresGlobalState globalState) { sb.append(globalState.getRandomly().getString().replace("'", "''")); sb.append("'"); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } - public static Query createListen() { + public static SQLQueryAdapter createListen() { StringBuilder sb = new StringBuilder(); sb.append("LISTEN "); sb.append(getChannel()); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } - public static Query createUnlisten() { + public static SQLQueryAdapter createUnlisten() { StringBuilder sb = new StringBuilder(); sb.append("UNLISTEN "); if (Randomly.getBoolean()) { @@ -42,7 +41,7 @@ public static Query createUnlisten() { } else { sb.append("*"); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java b/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java deleted file mode 100644 index 2746cc814..000000000 --- a/src/sqlancer/postgres/gen/PostgresQueryCatalogGenerator.java +++ /dev/null @@ -1,52 +0,0 @@ -package sqlancer.postgres.gen; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; - -import sqlancer.GlobalState; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.Randomly; - -public final class PostgresQueryCatalogGenerator { - - private PostgresQueryCatalogGenerator() { - } - - public static Query query() { - StringBuilder sb = new StringBuilder(); - sb.append("SELECT * FROM "); - sb.append( - Randomly.fromOptions("pg_aggregate", "pg_am", "pg_amop", "pg_amproc", "pg_attrdef", "pg_attribute", - "pg_authid", "pg_auth_members", "pg_cast", "pg_class", "pg_collation", "pg_constraint", - "pg_conversion", "pg_database", "pg_db_role_setting", "pg_default_acl", "pg_depend", - "pg_description", "pg_enum", "pg_event_trigger", "pg_extension", "pg_foreign_data_wrapper", - "pg_foreign_server", "pg_foreign_table", "pg_index", "pg_inherits", "pg_init_privs", - "pg_language", "pg_largeobject", "pg_largeobject_metadata", "pg_namespace", "pg_opclass", - "pg_operator", "pg_opfamily", "pg_partitioned_table", /* "pg_pltemplate", */ - "pg_policy", "pg_proc", "pg_publication", "pg_publication_rel", "pg_range", - "pg_replication_origin", "pg_rewrite", "pg_seclabel", "pg_sequence", "pg_shdepend", - "pg_shdescription", "pg_shseclabel", "pg_statistic", "pg_statistic_ext", - "pg_statistic_ext_data", "pg_subscription", "pg_subscription_rel", "pg_tablespace", - "pg_transform", "pg_trigger", "pg_ts_config", "pg_ts_config_map", "pg_ts_dict", "pg_ts_parser", - "pg_ts_template", "pg_type", "pg_user_mapping")); - return new QueryAdapter(sb.toString()) { - @Override - public boolean execute(GlobalState globalState) throws SQLException { - try (Statement s = globalState.getConnection().createStatement()) { - try (ResultSet rs = s.executeQuery(getQueryString())) { - // CHECKSTYLE:OFF - while (rs.next()) { - // only force the DBMS to fetch the records and hope that they cause an - // invalid state - } - // CHECKSTYLE:ON - } - } - return true; - } - }; - } - -} diff --git a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java index d1527ca8b..64d6278de 100644 --- a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static PostgresSelect createRandomQuery(int nrColumns, PostgresGlobalStat } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index cd3480512..d22ffe53e 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -1,13 +1,11 @@ package sqlancer.postgres.gen; -import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresIndex; @@ -20,8 +18,8 @@ private enum Scope { INDEX, TABLE, DATABASE; } - public static Query create(PostgresGlobalState globalState) { - List errors = new ArrayList<>(); + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); errors.add("could not create unique index"); // CONCURRENT INDEX StringBuilder sb = new StringBuilder(); sb.append("REINDEX"); @@ -40,7 +38,7 @@ public static Query create(PostgresGlobalState globalState) { if (indexes.isEmpty()) { throw new IgnoreMeException(); } - sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining())); + sb.append(Randomly.fromList(indexes).getIndexName()); break; case TABLE: sb.append("TABLE "); @@ -60,9 +58,8 @@ public static Query create(PostgresGlobalState globalState) { throw new AssertionError(scope); } errors.add("already contains data"); // FIXME bug report - errors.add("does not exist"); // internal index errors.add("REINDEX is not yet implemented for partitioned indexes"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java index e21d456d1..281e89e3d 100644 --- a/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSequenceGenerator.java @@ -1,11 +1,8 @@ package sqlancer.postgres.gen; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresSequenceGenerator { @@ -13,8 +10,8 @@ public final class PostgresSequenceGenerator { private PostgresSequenceGenerator() { } - public static Query createSequence(PostgresGlobalState globalState) { - List errors = new ArrayList<>(); + public static SQLQueryAdapter createSequence(PostgresGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE"); if (Randomly.getBoolean()) { sb.append(" "); @@ -87,7 +84,7 @@ public static Query createSequence(PostgresGlobalState globalState) { // sb.append(s.getRandomTable().getRandomColumn().getFullQualifiedName()); // } } - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index d403f0e0a..38318b26b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -4,9 +4,8 @@ import java.util.Arrays; import java.util.function.Function; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresSetGenerator { @@ -15,7 +14,7 @@ private PostgresSetGenerator() { } private enum ConfigurationOption { - // https://www.postgresql.org/docs/11/runtime-config-wal.html + // https://www.postgresql.org/docs/13/runtime-config-wal.html // This parameter can only be set at server start. // WAL_LEVEL("wal_level", (r) -> Randomly.fromOptions("replica", "minimal", "logical")), // FSYNC("fsync", (r) -> Randomly.fromOptions(1, 0)), @@ -38,7 +37,7 @@ private enum ConfigurationOption { // archive_mode // archive_command // archive_timeout - // https://www.postgresql.org/docs/11/runtime-config-statistics.html + // https://www.postgresql.org/docs/13/runtime-config-statistics.html // 19.9.1. Query and Index Statistics Collector TRACK_ACTIVITIES("track_activities", (r) -> Randomly.fromOptions(1, 0)), // track_activity_query_size @@ -47,7 +46,7 @@ private enum ConfigurationOption { TRACK_FUNCTIONS("track_functions", (r) -> Randomly.fromOptions("'none'", "'pl'", "'all'")), // stats_temp_directory // TODO 19.9.2. Statistics Monitoring - // https://www.postgresql.org/docs/11/runtime-config-autovacuum.html + // https://www.postgresql.org/docs/13/runtime-config-autovacuum.html // all can only be set at server-conf time // 19.11. Client Connection Defaults VACUUM_FREEZE_TABLE_AGE("vacuum_freeze_table_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), @@ -56,18 +55,14 @@ private enum ConfigurationOption { (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), VACUUM_MULTIXACT_FREEZE_MIN_AGE("vacuum_multixact_freeze_min_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), - VACUUM_CLEANUP_INDEX_SCALE_FACTOR("vacuum_cleanup_index_scale_factor", - (r) -> Randomly.fromOptions(0.0, 0.0000001, 0.00001, 0.01, 0.1, 1, 10, 100, 100000, 10000000000.0)), // TODO others GIN_FUZZY_SEARCH_LIMIT("gin_fuzzy_search_limit", (r) -> r.getInteger(0, 2147483647)), // 19.13. Version and Platform Compatibility DEFAULT_WITH_OIDS("default_with_oids", (r) -> Randomly.fromOptions(0, 1)), SYNCHRONIZED_SEQSCANS("synchronize_seqscans", (r) -> Randomly.fromOptions(0, 1)), - // https://www.postgresql.org/docs/devel/runtime-config-query.html + // https://www.postgresql.org/docs/13/runtime-config-query.html ENABLE_BITMAPSCAN("enable_bitmapscan", (r) -> Randomly.fromOptions(1, 0)), ENABLE_GATHERMERGE("enable_gathermerge", (r) -> Randomly.fromOptions(1, 0)), - ENABLE_HASHAGG("enable_hashagg", (r) -> Randomly.fromOptions(1, 0)), - ENABLE_HASHAGG_DISK("enable_hashagg_disk", (r) -> Randomly.fromOptions(1, 0)), ENABLE_HASHJOIN("enable_hashjoin", (r) -> Randomly.fromOptions(1, 0)), ENABLE_INDEXSCAN("enable_indexscan", (r) -> Randomly.fromOptions(1, 0)), ENABLE_INDEXONLYSCAN("enable_indexonlyscan", (r) -> Randomly.fromOptions(1, 0)), @@ -83,7 +78,7 @@ private enum ConfigurationOption { ENABLE_SORT("enable_sort", (r) -> Randomly.fromOptions(1, 0)), ENABLE_TIDSCAN("enable_tidscan", (r) -> Randomly.fromOptions(1, 0)), // 19.7.2. Planner Cost Constants (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS SEQ_PAGE_COST("seq_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), RANDOM_PAGE_COST("random_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), CPU_TUPLE_COST("cpu_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), @@ -99,7 +94,7 @@ private enum ConfigurationOption { JIT_OPTIMIZE_ABOVE_COST("jit_optimize_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), // 19.7.3. Genetic Query Optimizer (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO GEQO("geqo", (r) -> Randomly.fromOptions(1, 0)), GEQO_THRESHOLD("geqo_threshold", (r) -> r.getInteger(2, 2147483647)), GEQO_EFFORT("geqo_effort", (r) -> r.getInteger(1, 10)), @@ -108,7 +103,7 @@ private enum ConfigurationOption { GEQO_SELECTION_BIAS("geqo_selection_bias", (r) -> Randomly.fromOptions(1.5, 1.8, 2.0)), GEQO_SEED("geqo_seed", (r) -> Randomly.fromOptions(0, 0.5, 1)), // 19.7.4. Other Planner Options (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER DEFAULT_STATISTICS_TARGET("default_statistics_target", (r) -> r.getInteger(1, 10000)), CONSTRAINT_EXCLUSION("constraint_exclusion", (r) -> Randomly.fromOptions("on", "off", "partition")), CURSOR_TUPLE_FRACTION("cursor_tuple_fraction", @@ -117,7 +112,7 @@ private enum ConfigurationOption { JIT("jit", (r) -> Randomly.fromOptions(1, 0)), JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), - FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), + // FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), PLAN_CACHE_MODE("plan_cache_mode", (r) -> Randomly.fromOptions("auto", "force_generic_plan", "force_custom_plan")); @@ -130,7 +125,7 @@ private enum ConfigurationOption { } } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); ArrayList options = new ArrayList<>(Arrays.asList(ConfigurationOption.values())); options.remove(ConfigurationOption.DEFAULT_WITH_OIDS); @@ -147,7 +142,7 @@ public static Query create(PostgresGlobalState globalState) { } else { sb.append(option.op.apply(globalState.getRandomly())); } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java index 90b30d41e..9a82e29af 100644 --- a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java @@ -1,13 +1,12 @@ package sqlancer.postgres.gen; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresStatisticsObject; @@ -18,7 +17,7 @@ public final class PostgresStatisticsGenerator { private PostgresStatisticsGenerator() { } - public static Query insert(PostgresGlobalState globalState) { + public static SQLQueryAdapter insert(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("CREATE STATISTICS "); if (Randomly.getBoolean()) { @@ -44,10 +43,11 @@ public static Query insert(PostgresGlobalState globalState) { sb.append(randomColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(" FROM "); sb.append(randomTable.getName()); - return new QueryAdapter(sb.toString(), Arrays.asList("cannot have more than 8 columns in statistics"), true); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot have more than 8 columns in statistics"), + true); } - public static Query remove(PostgresGlobalState globalState) { + public static SQLQueryAdapter remove(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder("DROP STATISTICS "); PostgresTable randomTable = globalState.getSchema().getRandomTable(); List statistics = randomTable.getStatistics(); @@ -55,7 +55,21 @@ public static Query remove(PostgresGlobalState globalState) { throw new IgnoreMeException(); } sb.append(Randomly.fromList(statistics).getName()); - return new QueryAdapter(sb.toString(), true); + return new SQLQueryAdapter(sb.toString(), true); + } + + public static SQLQueryAdapter alter(PostgresGlobalState globalState) { + StringBuilder sb = new StringBuilder("ALTER STATISTICS "); + PostgresTable randomTable = globalState.getSchema().getRandomTable(); + List statistics = randomTable.getStatistics(); + if (statistics.isEmpty()) { + throw new IgnoreMeException(); + } + PostgresStatisticsObject randomStatistic = Randomly.fromList(statistics); + sb.append(randomStatistic.getName()); + sb.append(" SET STATISTICS "); + sb.append(Randomly.getNotCachedInteger(-1, 10000)); // -1 means default + return new SQLQueryAdapter(sb.toString(), true); } private static String getNewStatisticsName(PostgresTable randomTable) { diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 68226d1dd..9a7bfb032 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -1,14 +1,13 @@ package sqlancer.postgres.gen; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -16,7 +15,6 @@ import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.sqlite3.gen.SQLite3Common; public class PostgresTableGenerator { @@ -25,9 +23,10 @@ public class PostgresTableGenerator { private boolean columnHasPrimaryKey; private final StringBuilder sb = new StringBuilder(); private boolean isTemporaryTable; + private boolean isPartitionedTable; private final PostgresSchema newSchema; private final List columnsToBeAdded = new ArrayList<>(); - private final Set errors = new HashSet<>(); + protected final ExpectedErrors errors = new ExpectedErrors(); private final PostgresTable table; private final boolean generateOnlyKnown; private final PostgresGlobalState globalState; @@ -53,16 +52,19 @@ public PostgresTableGenerator(String tableName, PostgresSchema newSchema, boolea errors.add("does not accept data type"); errors.add("but default expression is of type text"); errors.add("has pseudo-type unknown"); + errors.add("no collation was derived for partition key column"); + errors.add("inherits from generated column but specifies identity"); + errors.add("inherits from generated column but specifies default"); PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonTableErrors(errors); } - public static Query generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, + public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, PostgresGlobalState globalState) { return new PostgresTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); } - private Query generate() { + protected SQLQueryAdapter generate() { columnCanHavePrimaryKey = true; sb.append("CREATE"); if (Randomly.getBoolean()) { @@ -83,7 +85,7 @@ private Query generate() { } else { createStandard(); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } private void createStandard() throws AssertionError { @@ -92,7 +94,7 @@ private void createStandard() throws AssertionError { if (i != 0) { sb.append(", "); } - String name = SQLite3Common.createColumnName(i); + String name = DBMSCommon.createColumnName(i); createColumn(name); } if (Randomly.getBoolean()) { @@ -110,7 +112,10 @@ private void createStandard() throws AssertionError { sb.append(")"); generateInherits(); generatePartitionBy(); - PostgresCommon.generateWith(sb, globalState, errors); + generateUsing(); + if (!isPartitionedTable) { + PostgresCommon.generateWith(sb, globalState, errors); + } if (Randomly.getBoolean() && isTemporaryTable) { sb.append(" ON COMMIT "); sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS", "DROP")); @@ -151,8 +156,10 @@ private void createColumn(String name) throws AssertionError { private void generatePartitionBy() { if (Randomly.getBoolean()) { + isPartitionedTable = false; return; } + isPartitionedTable = true; sb.append(" PARTITION BY "); // TODO "RANGE", String partitionOption = Randomly.fromOptions("RANGE", "LIST", "HASH"); @@ -184,8 +191,23 @@ private void generatePartitionBy() { sb.append(")"); } - private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { + private void generateUsing() { + /* + * Postgres does not allow specifying USING clause for partitioned tables since they don't have any storage + * associated with them + */ + if (isPartitionedTable) { + return; + } + if (Randomly.getBoolean()) { + return; + } + sb.append(" USING "); + sb.append(globalState.getRandomTableAccessMethod()); + } + + protected void generateInherits() { + if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); @@ -196,6 +218,8 @@ private void generateInherits() { errors.add("cannot inherit from temporary relation"); errors.add("cannot inherit from partitioned table"); errors.add("has a collation conflict"); + errors.add("inherits conflicting default values"); + errors.add("specifies generation expression"); } } @@ -232,6 +256,7 @@ private void createColumnConstraint(PostgresDataType type, boolean serial) { switch (c) { case NULL_OR_NOT_NULL: sb.append(Randomly.fromOptions("NOT NULL", "NULL")); + errors.add("conflicting NULL/NOT NULL declarations"); break; case UNIQUE: sb.append("UNIQUE"); diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java new file mode 100644 index 000000000..1be99ca61 --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -0,0 +1,58 @@ +package sqlancer.postgres.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; + +public class PostgresTableSpaceGenerator { + + private final ExpectedErrors errors = new ExpectedErrors(); + private final PostgresGlobalState globalState; + + public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; + errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace\\d+\" does not exist"); + errors.add("ERROR: already exists"); + errors.add("ERROR: is not empty"); + errors.add("ERROR: cannot be created because system does not support tablespaces"); + } + + public static SQLQueryAdapter generate(PostgresGlobalState globalState) { + // PostgresProvider.mapActions does not schedule this action when the option is disabled, but QPG + // selects actions by index without consulting the schedule, so the generator has to report that + // it has nothing to generate. + PostgresOptions options = globalState.getDbmsSpecificOptions(); + if (!options.isTestTablespaces()) { + throw new IgnoreMeException(); + } + return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); + } + + private SQLQueryAdapter generateTableSpace() { + StringBuilder sb = new StringBuilder(); + int tableSpaceNum = globalState.getRandomly().getInteger(1, Integer.MAX_VALUE); + + // CREATE TABLESPACE syntax + sb.append("CREATE TABLESPACE "); + sb.append("tablespace"); + sb.append(tableSpaceNum); + sb.append(" LOCATION '"); + + // Get the validated base path from options and append the tablespace number + PostgresOptions options = globalState.getDbmsSpecificOptions(); + String path = options.getTablespacePath() + tableSpaceNum; + + // Convert backslashes to forward slashes for PostgreSQL + path = path.replace('\\', '/'); + + // Escape single quotes in the path + path = path.replace("'", "''"); + + sb.append(path); + sb.append("'"); + + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java index 2c448e5d8..d58f37d3f 100644 --- a/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTransactionGenerator.java @@ -1,19 +1,16 @@ package sqlancer.postgres.gen; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; public final class PostgresTransactionGenerator { private PostgresTransactionGenerator() { } - public static Query executeBegin() { - List errors = new ArrayList<>(); + public static SQLQueryAdapter executeBegin() { + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("BEGIN"); if (Randomly.getBoolean()) { errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); @@ -24,7 +21,7 @@ public static Query executeBegin() { // sb.append(Randomly.fromOptions("READ WRITE", "READ ONLY")); // } } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java index ebcb22ad9..0745d1cce 100644 --- a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java @@ -1,11 +1,10 @@ package sqlancer.postgres.gen; -import java.util.Arrays; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; public final class PostgresTruncateGenerator { @@ -13,7 +12,7 @@ public final class PostgresTruncateGenerator { private PostgresTruncateGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); if (Randomly.getBoolean()) { @@ -34,8 +33,9 @@ public static Query create(PostgresGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new QueryAdapter(sb.toString(), - Arrays.asList("cannot truncate a table referenced in a foreign key constraint", "is not a table")); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot truncate a table referenced in a foreign key constraint", "is not a table", + "is not distributed")); } } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 16d0cb9e3..92257a95a 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -1,13 +1,11 @@ package sqlancer.postgres.gen; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -15,50 +13,38 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public final class PostgresUpdateGenerator { +public final class PostgresUpdateGenerator extends AbstractUpdateGenerator { - private PostgresUpdateGenerator() { - } + private final PostgresGlobalState globalState; + private PostgresTable randomTable; - public static Query create(PostgresGlobalState globalState) { - PostgresTable randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); - StringBuilder sb = new StringBuilder(); - sb.append("UPDATE "); - sb.append(randomTable.getName()); - sb.append(" SET "); - Set errors = new HashSet<>(Arrays.asList("conflicting key value violates exclusion constraint", + private PostgresUpdateGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", " bit string too long", "can only be updated to DEFAULT", "division by zero", "You might need to add explicit type casts.", "invalid regular expression", "View columns that are not columns of their base relation are not updatable")); + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + return new PostgresUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + sb.append("UPDATE "); + sb.append(randomTable.getName()); + sb.append(" SET "); errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced // table multiple times - List columns = randomTable.getRandomNonEmptyColumnSubset(); + errors.add("new row violates check option for view"); PostgresCommon.addCommonInsertUpdateErrors(errors); - - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - PostgresColumn column = columns.get(i); - sb.append(column.getName()); - sb.append(" = "); - if (!Randomly.getBoolean()) { - PostgresExpression constant = PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), - column.getType()); - sb.append(PostgresVisitor.asString(constant)); - } else if (Randomly.getBoolean()) { - sb.append("DEFAULT"); - } else { - sb.append("("); - PostgresExpression expr = PostgresExpressionGenerator.generateExpression(globalState, - randomTable.getColumns(), column.getType()); - // caused by casts - sb.append(PostgresVisitor.asString(expr)); - sb.append(")"); - } - } + updateColumns(columns); errors.add("invalid input syntax for "); errors.add("operator does not exist: text = boolean"); errors.add("violates check constraint"); @@ -66,13 +52,28 @@ public static Query create(PostgresGlobalState globalState) { errors.add("but expression is of type"); PostgresCommon.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); PostgresExpression where = PostgresExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), PostgresDataType.BOOLEAN); - sb.append(PostgresVisitor.asString(where)); + appendWhereClause(PostgresVisitor.asString(where)); } + } - return new QueryAdapter(sb.toString(), errors, true); + @Override + protected void updateValue(PostgresColumn column) { + if (!Randomly.getBoolean()) { + PostgresExpression constant = PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(PostgresVisitor.asString(constant)); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append("("); + PostgresExpression expr = PostgresExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), column.getType()); + // caused by casts + sb.append(PostgresVisitor.asString(expr)); + sb.append(")"); + } } } diff --git a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java index 9571af351..ef32db45a 100644 --- a/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresVacuumGenerator.java @@ -2,12 +2,11 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresTable; @@ -16,7 +15,7 @@ public final class PostgresVacuumGenerator { private PostgresVacuumGenerator() { } - public static Query create(PostgresGlobalState globalState) { + public static SQLQueryAdapter create(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(); StringBuilder sb = new StringBuilder("VACUUM "); if (Randomly.getBoolean()) { @@ -53,7 +52,7 @@ public static Query create(PostgresGlobalState globalState) { } } } - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); errors.add("VACUUM cannot run inside a transaction block"); errors.add("deadlock"); /* * "FULL" commented out due to https://www.postgresql.org/message-id/CA%2Bu7OA6pL% @@ -61,7 +60,7 @@ public static Query create(PostgresGlobalState globalState) { */ errors.add("ERROR: ANALYZE option must be specified when a column list is provided"); errors.add("VACUUM option DISABLE_PAGE_SKIPPING cannot be used with FULL"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } private static void addTableAndColumns(PostgresTable table, StringBuilder sb) { diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index afc91d341..10992ece6 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -1,23 +1,20 @@ package sqlancer.postgres.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.sqlite3.gen.SQLite3Common; public final class PostgresViewGenerator { private PostgresViewGenerator() { } - public static Query create(PostgresGlobalState globalState) { - Set errors = new HashSet<>(); + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE"); boolean materialized; boolean recursive = false; @@ -38,23 +35,15 @@ public static Query create(PostgresGlobalState globalState) { materialized = false; } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } - sb.append(SQLite3Common.createColumnName(i)); + sb.append(DBMSCommon.createColumnName(i)); } sb.append(")"); // if (Randomly.getBoolean() && false) { @@ -89,7 +78,7 @@ public static Query create(PostgresGlobalState globalState) { errors.add("non-integer constant in DISTINCT ON"); errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); PostgresCommon.addCommonExpressionErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java new file mode 100644 index 000000000..6daf66c3c --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -0,0 +1,131 @@ +package sqlancer.postgres.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.ast.PostgresConstant; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresOrderByTerm; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; + +public final class PostgresWindowFunctionGenerator { + + private static final List WINDOW_FUNCTIONS = Arrays.asList("row_number", "rank", "dense_rank", + "percent_rank", "cume_dist", "ntile", "lag", "lead", "first_value", "last_value", "nth_value"); + + private PostgresWindowFunctionGenerator() { + throw new AssertionError("Utility class should not be instantiated"); + } + + public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, + List availableExpr) { + + String functionName = selectRandomWindowFunction(); + List arguments = generateFunctionArguments(functionName, globalState, availableExpr); + WindowSpecification windowSpec = generateWindowSpecification(globalState, availableExpr); + PostgresDataType returnType = determineReturnType(functionName); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } + + private static String selectRandomWindowFunction() { + return Randomly.fromList(WINDOW_FUNCTIONS); + } + + private static List generateFunctionArguments(String functionName, + PostgresGlobalState globalState, List availableExpr) { + List arguments = new ArrayList<>(); + + switch (functionName) { + case "ntile": + arguments + .add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); + break; + case "lag": + case "lead": + case "nth_value": + arguments.add(Randomly.fromList(availableExpr)); + if (Randomly.getBoolean()) { + arguments.add( + PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); + } + break; + case "first_value": + case "last_value": + arguments.add(Randomly.fromList(availableExpr)); + break; + default: + // No arguments needed for other window functions + break; + } + + return arguments; + } + + private static WindowSpecification generateWindowSpecification(PostgresGlobalState globalState, + List availableExpr) { + List partitionBy = generatePartitionByClause(availableExpr); + PostgresExpressionGenerator exprGen = new PostgresExpressionGenerator(globalState); + List orderBys = exprGen.generateOrderBys(); + List orderByTerms = new ArrayList<>(); + for (PostgresExpression expr : orderBys) { + orderByTerms.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + } + + WindowFrame frame = generateWindowFrame(globalState); + return new WindowSpecification(partitionBy, orderByTerms, frame); + } + + private static List generatePartitionByClause(List availableExpr) { + List partitionBy = new ArrayList<>(); + if (Randomly.getBooleanWithRatherLowProbability()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + partitionBy.add(Randomly.fromList(availableExpr)); + } + } + return partitionBy; + } + + private static WindowFrame generateWindowFrame(PostgresGlobalState globalState) { + if (Randomly.getBooleanWithRatherLowProbability()) { + WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); + PostgresExpression startExpr = generateFrameBound(globalState); + PostgresExpression endExpr = generateFrameBound(globalState); + return new WindowFrame(frameType, startExpr, endExpr); + } + return null; + } + + private static PostgresExpression generateFrameBound(PostgresGlobalState globalState) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateCurrentRowBound(); + } else { + return generateOffsetBound(globalState); + } + } + + private static PostgresExpression generateCurrentRowBound() { + return PostgresConstant.createIntConstant(0); + } + + private static PostgresExpression generateOffsetBound(PostgresGlobalState globalState) { + return PostgresConstant.createIntConstant(globalState.getRandomly().getInteger()); + } + + private static PostgresDataType determineReturnType(String functionName) { + switch (functionName) { + case "percent_rank": + case "cume_dist": + return PostgresDataType.FLOAT; + default: + return PostgresDataType.INT; + } + } +} diff --git a/src/sqlancer/postgres/oracle/PostgresFuzzer.java b/src/sqlancer/postgres/oracle/PostgresFuzzer.java new file mode 100644 index 000000000..a0b46bd17 --- /dev/null +++ b/src/sqlancer/postgres/oracle/PostgresFuzzer.java @@ -0,0 +1,30 @@ +package sqlancer.postgres.oracle; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.gen.PostgresRandomQueryGenerator; + +public class PostgresFuzzer implements TestOracle { + + private final PostgresGlobalState globalState; + + public PostgresFuzzer(PostgresGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public void check() throws Exception { + String s = PostgresVisitor.asString( + PostgresRandomQueryGenerator.createRandomQuery(Randomly.smallNumber() + 1, globalState)) + ';'; + try { + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); + } catch (Error e) { + + } + } + +} diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java deleted file mode 100644 index b58e1cbed..000000000 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ /dev/null @@ -1,170 +0,0 @@ -package sqlancer.postgres.oracle; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.Randomly; -import sqlancer.StateToReproduce.PostgresStateToReproduce; -import sqlancer.TestOracle; -import sqlancer.postgres.PostgresCompoundDataType; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresDataType; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.PostgresVisitor; -import sqlancer.postgres.ast.PostgresCastOperation; -import sqlancer.postgres.ast.PostgresColumnValue; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresPostfixText; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; -import sqlancer.postgres.ast.PostgresSelect.SelectType; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class PostgresNoRECOracle implements TestOracle { - - private final PostgresSchema s; - private final Connection con; - private final PostgresStateToReproduce state; - private String firstQueryString; - private String secondQueryString; - private final StateLogger logger; - private final MainOptions options; - private final Set errors = new HashSet<>(); - private final PostgresGlobalState globalState; - - public PostgresNoRECOracle(PostgresGlobalState globalState) { - this.s = globalState.getSchema(); - this.con = globalState.getConnection(); - this.state = (PostgresStateToReproduce) globalState.getState(); - this.logger = globalState.getLogger(); - this.options = globalState.getOptions(); - this.globalState = globalState; - } - - @Override - public void check() throws SQLException { - PostgresCommon.addCommonExpressionErrors(errors); - PostgresCommon.addCommonFetchErrors(errors); - PostgresTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - PostgresExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); - - List joinStatements = getJoinStatements(globalState, columns, tables); - List fromTables = tables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) - .collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - state.queryString = firstCount + " " + secondCount + " " + firstQueryString + ";\n" + secondQueryString - + ";"; - throw new AssertionError(firstQueryString + secondQueryString + firstCount + " " + secondCount); - } - } - - public static List getJoinStatements(PostgresGlobalState globalState, List columns, - List tables) { - List joinStatements = new ArrayList<>(); - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); - PostgresTable table = Randomly.fromList(tables); - tables.remove(table); - PostgresJoinType options = PostgresJoinType.getRandom(); - PostgresJoin j = new PostgresJoin(table, joinClause, options); - joinStatements.add(j); - } - return joinStatements; - } - - private PostgresExpression getRandomWhereCondition(List columns) { - return new PostgresExpressionGenerator(globalState).setColumns(columns).setGlobalState(globalState) - .generateExpression(PostgresDataType.BOOLEAN); - } - - private int getUnoptimizedQueryCount(List fromTables, PostgresExpression randomWhereCondition, - List joinStatements) throws SQLException { - PostgresSelect select = new PostgresSelect(); - PostgresCastOperation isTrue = new PostgresCastOperation(randomWhereCondition, - PostgresCompoundDataType.create(PostgresDataType.INT)); - PostgresPostfixText asText = new PostgresPostfixText(isTrue, " as count", null, PostgresDataType.INT); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(fromTables); - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - secondQueryString = "SELECT SUM(count) FROM (" + PostgresVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(secondQueryString); - } - errors.add("canceling statement due to statement timeout"); - Query q = new QueryAdapter(secondQueryString, errors); - ResultSet rs; - try { - rs = q.executeAndGet(globalState); - } catch (Exception e) { - throw new AssertionError(secondQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(List randomTables, List columns, - PostgresExpression randomWhereCondition, List joinStatements) throws SQLException { - PostgresSelect select = new PostgresSelect(); - PostgresColumnValue allColumns = new PostgresColumnValue(Randomly.fromList(columns), null); - select.setFetchColumns(Arrays.asList(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(new PostgresExpressionGenerator(globalState).setColumns(columns) - .setGlobalState(globalState).generateOrderBy()); - } - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - firstQueryString = PostgresVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(firstQueryString); - } - try (ResultSet rs = stat.executeQuery(firstQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; - } - -} diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index 488b29054..9db5365a3 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -1,89 +1,60 @@ package sqlancer.postgres.oracle; -import java.sql.Connection; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import org.postgresql.util.PSQLException; - -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; import sqlancer.Randomly; -import sqlancer.StateToReproduce.PostgresStateToReproduce; -import sqlancer.TestOracle; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresRowValue; import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.PostgresToStringVisitor; +import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresPostfixOperation; +import sqlancer.postgres.ast.PostgresPostfixOperation.PostfixOperator; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class PostgresPivotedQuerySynthesisOracle implements TestOracle { +public class PostgresPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { - private PostgresStateToReproduce state; - private PostgresRowValue rw; - private final Connection database; private List fetchColumns; - private final PostgresSchema s; - private final MainOptions options; - private final StateLogger logger; - private final PostgresGlobalState globalState; public PostgresPivotedQuerySynthesisOracle(PostgresGlobalState globalState) throws SQLException { - this.globalState = globalState; - this.database = globalState.getConnection(); - this.s = globalState.getSchema(); - options = globalState.getOptions(); - logger = globalState.getLogger(); + super(globalState); + PostgresCommon.addCommonExpressionErrors(errors); + PostgresCommon.addCommonFetchErrors(errors); } @Override - public void check() throws SQLException { - String queryString = getQueryThatContainsAtLeastOneRow(state); - state.queryString = queryString; - if (options.logEachSelect()) { - logger.writeCurrent(state.queryString); - } - - boolean isContainedIn = isContainedIn(queryString, options, logger); - if (!isContainedIn) { - throw new AssertionError(queryString); - } - - } - - public String getQueryThatContainsAtLeastOneRow(PostgresStateToReproduce state) throws SQLException { - this.state = state; - PostgresTables randomFromTables = s.getRandomTableNonEmptyTables(); - - state.queryTargetedTablesString = randomFromTables.tableNamesAsString(); + public SQLQueryAdapter getRectifiedQuery() throws SQLException { + PostgresTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); PostgresSelect selectStatement = new PostgresSelect(); selectStatement.setSelectType(Randomly.fromOptions(PostgresSelect.SelectType.values())); List columns = randomFromTables.getColumns(); - rw = randomFromTables.getRandomRowValue(database, state); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); fetchColumns = columns; selectStatement.setFromList(randomFromTables.getTables().stream().map(t -> new PostgresFromTable(t, false)) .collect(Collectors.toList())); selectStatement.setFetchColumns(fetchColumns.stream() - .map(c -> new PostgresColumnValue(c, rw.getValues().get(c))).collect(Collectors.toList())); - state.queryTargetedColumnsString = fetchColumns.stream().map(c -> c.getFullQualifiedName()) - .collect(Collectors.joining(", ")); - PostgresExpression whereClause = generateWhereClauseThatContainsRowValue(columns, rw); + .map(c -> new PostgresColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + PostgresExpression whereClause = generateRectifiedExpression(columns, pivotRow); selectStatement.setWhereClause(whereClause); - state.whereClause = selectStatement; - List groupByClause = generateGroupByClause(columns, rw); + List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); PostgresExpression limitClause = generateLimit(); selectStatement.setLimitClause(limitClause); @@ -92,32 +63,19 @@ public String getQueryThatContainsAtLeastOneRow(PostgresStateToReproduce state) selectStatement.setOffsetClause(offsetClause); } List orderBy = new PostgresExpressionGenerator(globalState).setColumns(columns) - .generateOrderBy(); - selectStatement.setOrderByExpressions(orderBy); - - StringBuilder sb2 = new StringBuilder(); - sb2.append("SELECT * FROM (SELECT 1 FROM "); - sb2.append(randomFromTables.tableNamesAsString()); - sb2.append(" WHERE "); - int i = 0; - for (PostgresColumn c : fetchColumns) { - if (i++ != 0) { - sb2.append(" AND "); - } - sb2.append(c.getFullQualifiedName()); - if (rw.getValues().get(c).isNull()) { - sb2.append(" IS NULL"); - } else { - sb2.append(" = "); - sb2.append(rw.getValues().get(c).getTextRepresentation()); - } - } - sb2.append(") as result;"); - state.queryThatSelectsRow = sb2.toString(); + .generateOrderBys(); + selectStatement.setOrderByClauses(orderBy); + return new SQLQueryAdapter(PostgresVisitor.asString(selectStatement)); + } - PostgresToStringVisitor visitor = new PostgresToStringVisitor(); - visitor.visit(selectStatement); - return visitor.get(); + /* + * Prevent name collisions by aliasing the column. + */ + private PostgresColumn getFetchValueAliasedColumn(PostgresColumn c) { + PostgresColumn aliasedColumn = new PostgresColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType()); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; } private List generateGroupByClause(List columns, PostgresRowValue rw) { @@ -139,63 +97,55 @@ private PostgresConstant generateLimit() { private PostgresExpression generateOffset() { if (Randomly.getBoolean()) { - // OFFSET 0 return PostgresConstant.createIntConstant(0); } else { return null; } } - private PostgresExpression generateWhereClauseThatContainsRowValue(List columns, - PostgresRowValue rw) { - return PostgresExpressionGenerator.generateTrueCondition(columns, rw, globalState); + private PostgresExpression generateRectifiedExpression(List columns, PostgresRowValue rw) { + PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(PostgresDataType.BOOLEAN); + PostgresExpression result; + if (expr.getExpectedValue().isNull()) { + result = PostgresPostfixOperation.create(expr, PostfixOperator.IS_NULL); + } else { + result = PostgresPostfixOperation.create(expr, + expr.getExpectedValue().cast(PostgresDataType.BOOLEAN).asBoolean() ? PostfixOperator.IS_TRUE + : PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; } - private boolean isContainedIn(String queryString, MainOptions options, StateLogger logger) throws SQLException { - Statement createStatement; - createStatement = database.createStatement(); - + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions - sb.append(queryString); + sb.append(query.getUnterminatedQueryString()); sb.append(") as result WHERE "); int i = 0; for (PostgresColumn c : fetchColumns) { if (i++ != 0) { sb.append(" AND "); } - sb.append("result." + c.getTable().getName() + c.getName()); - if (rw.getValues().get(c).isNull()) { + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { sb.append(" IS NULL"); } else { sb.append(" = "); - sb.append(rw.getValues().get(c).getTextRepresentation()); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); } } String resultingQueryString = sb.toString(); - state.queryString = resultingQueryString; - if (options.logEachSelect()) { - logger.writeCurrent(resultingQueryString); - } - try (ResultSet result = createStatement.executeQuery(resultingQueryString)) { - boolean isContainedIn = result.next(); - createStatement.close(); - return isContainedIn; - } catch (PSQLException e) { - if (e.getMessage().contains("out of range") || e.getMessage().contains("cannot cast") - || e.getMessage().contains("invalid input syntax for ") || e.getMessage().contains("must be type") - || e.getMessage().contains("operator does not exist") - || e.getMessage().contains("Could not choose a best candidate function.") - || e.getMessage().contains("division by zero") - || e.getMessage().contains("zero raised to a negative power is undefined") - || e.getMessage().contains("canceling statement due to statement timeout") - || e.getMessage().contains("operator is not unique") - || e.getMessage().contains("could not determine which collation to use for string comparison")) { - return true; - } else { - throw e; - } - } + return new SQLQueryAdapter(resultingQueryString, errors); + } + + @Override + protected String getExpectedValues(PostgresExpression expr) { + return PostgresVisitor.asExpectedValues(expr); } } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index bb1d95e08..dba8c0a6d 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -1,6 +1,6 @@ package sqlancer.postgres.oracle.tlp; -import java.sql.ResultSet; +import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -10,9 +10,10 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresVisitor; @@ -28,7 +29,7 @@ import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.gen.PostgresCommon; -public class PostgresTLPAggregateOracle extends PostgresTLPBase implements TestOracle { +public class PostgresTLPAggregateOracle extends PostgresTLPBase implements TestOracle { private String firstResult; private String secondResult; @@ -43,6 +44,10 @@ public PostgresTLPAggregateOracle(PostgresGlobalState state) { @Override public void check() throws SQLException { super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() throws SQLException { PostgresAggregateFunction aggregateFunction = Randomly.fromOptions(PostgresAggregateFunction.MAX, PostgresAggregateFunction.MIN, PostgresAggregateFunction.SUM, PostgresAggregateFunction.BIT_AND, PostgresAggregateFunction.BIT_OR, PostgresAggregateFunction.BOOL_AND, PostgresAggregateFunction.BOOL_OR, @@ -56,24 +61,27 @@ public void check() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = PostgresVisitor.asString(select); firstResult = getAggregateResult(originalQuery); metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); secondResult = getAggregateResult(metamorphicQuery); - state.getState().queryString = "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult - + "\n-- " + secondResult; - if (firstResult == null && secondResult != null - || firstResult != null && (!firstResult.contentEquals(secondResult) - && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { - if (secondResult.contains("Inf")) { + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { throw new IgnoreMeException(); // FIXME: average computation } - throw new AssertionError(); + String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, + secondQueryString); + throw new AssertionError(assertionMessage); } - } private String createMetamorphicUnionQuery(PostgresSelect select, PostgresAggregate aggregate, @@ -86,7 +94,7 @@ private String createMetamorphicUnionQuery(PostgresSelect select, PostgresAggreg PostgresSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); PostgresSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); PostgresSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate).toString() + " FROM ("; + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; metamorphicQuery += PostgresVisitor.asString(leftSelect) + " UNION ALL " + PostgresVisitor.asString(middleSelect) + " UNION ALL " + PostgresVisitor.asString(rightSelect); metamorphicQuery += ") as asdf"; @@ -94,9 +102,20 @@ private String createMetamorphicUnionQuery(PostgresSelect select, PostgresAggreg } private String getAggregateResult(String queryString) throws SQLException { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } String resultString; - QueryAdapter q = new QueryAdapter(queryString, errors); - try (ResultSet result = q.executeAndGet(state)) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } @@ -123,13 +142,16 @@ private List mapped(PostgresAggregate aggregate) { case MIN: return aliasArgs(Arrays.asList(aggregate)); // case AVG: - //// List arg = Arrays.asList(new PostgresCast(aggregate.getExpr().get(0), + //// List arg = Arrays.asList(new + // PostgresCast(aggregate.getExpr().get(0), // PostgresDataType.DECIMAL.get())); - // PostgresAggregate sum = new PostgresAggregate(PostgresAggregateFunction.SUM, aggregate.getExpr()); + // PostgresAggregate sum = new PostgresAggregate(PostgresAggregateFunction.SUM, + // aggregate.getExpr()); // PostgresCast count = new PostgresCast( // new PostgresAggregate(PostgresAggregateFunction.COUNT, aggregate.getExpr()), // PostgresDataType.DECIMAL.get()); - //// PostgresBinaryArithmeticOperation avg = new PostgresBinaryArithmeticOperation(sum, count, + //// PostgresBinaryArithmeticOperation avg = new + // PostgresBinaryArithmeticOperation(sum, count, // PostgresBinaryArithmeticOperator.DIV); // return aliasArgs(Arrays.asList(sum, count)); default: diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 22abc849b..55b5d2dcf 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -1,47 +1,43 @@ package sqlancer.postgres.oracle.tlp; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTables; import sqlancer.postgres.ast.PostgresColumnValue; +import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresPostfixOperation; -import sqlancer.postgres.ast.PostgresPostfixOperation.PostfixOperator; -import sqlancer.postgres.ast.PostgresPrefixOperation; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.ForClause; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.postgres.oracle.PostgresNoRECOracle; -public class PostgresTLPBase implements TestOracle { +public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { - final PostgresGlobalState state; - final Set errors = new HashSet<>(); - - PostgresSchema s; - PostgresTables targetTables; - PostgresExpressionGenerator gen; - PostgresSelect select; - PostgresExpression predicate; - PostgresPrefixOperation negatedPredicate; - PostgresPostfixOperation isNullPredicate; + protected PostgresSchema s; + protected PostgresTables targetTables; + protected PostgresExpressionGenerator gen; + protected PostgresSelect select; public PostgresTLPBase(PostgresGlobalState state) { - this.state = state; + super(state); PostgresCommon.addCommonExpressionErrors(errors); PostgresCommon.addCommonFetchErrors(errors); } @@ -50,31 +46,95 @@ public PostgresTLPBase(PostgresGlobalState state) { public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); - gen = new PostgresExpressionGenerator(state).setColumns(targetTables.getColumns()); - select = new PostgresSelect(); - select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List joins = PostgresNoRECOracle.getJoinStatements(state, targetTables.getColumns(), tables); + List joins = getJoinStatements(state, targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(PostgresGlobalState globalState, List columns, + List tables) { + List joinStatements = new ArrayList<>(); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresTable table = Randomly.fromList(tables); + tables.remove(table); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PostgresSubquery subquery = PostgresTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + protected void generateSelectBase(List tables, List joins) { List tableList = tables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) .collect(Collectors.toList()); - // TODO joins + gen = new PostgresExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new PostgresSelect(); + select.setFetchColumns(generateFetchColumns()); select.setFromList(tableList); select.setWhereClause(null); select.setJoinClauses(joins); - predicate = generatePredicate(); - negatedPredicate = new PostgresPrefixOperation(predicate, PostgresPrefixOperation.PrefixOperator.NOT); - isNullPredicate = new PostgresPostfixOperation(predicate, PostfixOperator.IS_NULL); if (Randomly.getBoolean()) { select.setForClause(ForClause.getRandom()); } } List generateFetchColumns() { - return Arrays.asList(new PostgresColumnValue(targetTables.getColumns().get(0), null)); + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new PostgresColumnValue(PostgresColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(targetTables.getColumns()); + for (PostgresColumn c : targetColumns) { + fetchColumns.add(new PostgresColumnValue(c, null)); + } + return fetchColumns; } - PostgresExpression generatePredicate() { - return gen.generateExpression(PostgresDataType.BOOLEAN); + @Override + protected ExpressionGenerator getGen() { + return gen; + } + + public static PostgresSubquery createSubquery(PostgresGlobalState globalState, String name, PostgresTables tables) { + List columns = new ArrayList<>(); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + PostgresSelect select = new PostgresSelect(); + select.setFromList(tables.getTables().stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return new PostgresSubquery(select, name); } } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java index 6635ed0fa..2f520bebb 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java @@ -22,6 +22,10 @@ public PostgresTLPHavingOracle(PostgresGlobalState state) { @Override public void check() throws SQLException { super.check(); + havingCheck(); + } + + protected void havingCheck() throws SQLException { if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(PostgresDataType.BOOLEAN)); } @@ -32,7 +36,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setHavingClause(predicate); String firstQueryString = PostgresVisitor.asString(select); @@ -48,7 +52,7 @@ public void check() throws SQLException { } @Override - PostgresExpression generatePredicate() { + protected PostgresExpression generatePredicate() { return gen.generateHavingClause(); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java deleted file mode 100644 index e8b8d3146..000000000 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java +++ /dev/null @@ -1,41 +0,0 @@ -package sqlancer.postgres.oracle.tlp; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresVisitor; - -public class PostgresTLPWhereOracle extends PostgresTLPBase { - - public PostgresTLPWhereOracle(PostgresGlobalState state) { - super(state); - } - - @Override - public void check() throws SQLException { - super.check(); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); - } - String originalQueryString = PostgresVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByExpressions(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = PostgresVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = PostgresVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = PostgresVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } -} diff --git a/src/sqlancer/presto/PrestoBugs.java b/src/sqlancer/presto/PrestoBugs.java new file mode 100644 index 000000000..b0eb3fe57 --- /dev/null +++ b/src/sqlancer/presto/PrestoBugs.java @@ -0,0 +1,20 @@ +package sqlancer.presto; + +public final class PrestoBugs { + + // https://github.com/prestodb/presto/issues/23324 + public static boolean bug23324 = true; + + // https://github.com/prestodb/presto/issues/23613 + public static boolean bug23613 = true; + + // https://github.com/prestodb/presto/issues/27608 + public static boolean bugVerifyError = true; + + // https://github.com/prestodb/presto/issues/27609 + public static boolean bugCompilerFailed = true; + + private PrestoBugs() { + } + +} diff --git a/src/sqlancer/presto/PrestoConstantUtils.java b/src/sqlancer/presto/PrestoConstantUtils.java new file mode 100644 index 000000000..6de744019 --- /dev/null +++ b/src/sqlancer/presto/PrestoConstantUtils.java @@ -0,0 +1,40 @@ +package sqlancer.presto; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +public final class PrestoConstantUtils { + + private PrestoConstantUtils() { + } + + public static String removeNoneAscii(String str) { + return str.replaceAll("[^\\x00-\\x7F]", ""); + } + + public static String removeNonePrintable(String str) { // All Control Char + return str.replaceAll("[\\p{C}]", ""); + } + + public static String removeOthersControlChar(String str) { // Some Control Char + return str.replaceAll("[\\p{Cntrl}\\p{Cc}\\p{Cf}\\p{Co}\\p{Cn}]", ""); + } + + public static String removeAllControlChars(String str) { + return removeOthersControlChar(removeNonePrintable(str)).replaceAll("[\\r\\n\\t]", ""); + } + + public static BigDecimal getDecimal(double val, int scale, int precision) { + int part = precision - scale; + // long part + long lng = (long) val; + // decimal places + double d1 = val - lng; + String xStr = Long.toString(lng); + String substring = xStr.substring(xStr.length() - part); + long newX = substring.isEmpty() ? 0 : Long.parseLong(substring); + double finalD = newX + d1; + return new BigDecimal(finalD).setScale(scale, RoundingMode.CEILING); + } + +} diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java new file mode 100644 index 000000000..dd2931976 --- /dev/null +++ b/src/sqlancer/presto/PrestoErrors.java @@ -0,0 +1,195 @@ +package sqlancer.presto; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class PrestoErrors { + + private PrestoErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getFunctionErrors()); + + // Presto errors + errors.add("cannot be applied to"); + errors.add("LIKE expression must evaluate to a varchar"); + errors.add("JOIN ON clause must evaluate to a boolean"); + // errors.add("Unexpected parameters"); + + // SELECT SUM(count) FROM (SELECT + // CAST((-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0000 + // IS NOT NULL AND + // -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0000) + // AS BIGINT)as count FROM t0) as res + errors.add("Decimal overflow"); + errors.add("long overflow"); + errors.add("multiplication overflow"); + errors.add("addition overflow"); + errors.add("subtraction overflow"); + + // cast + // errors.add("Cannot cast"); + errors.add("Value cannot be cast to"); + errors.add("Cannot cast DECIMAL"); + errors.add("Cannot cast BIGINT"); + errors.add("Cannot cast INTEGER"); + + // TODO: check + errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); + errors.add("class io.airlift.slice.Slice cannot be cast to class java.lang.Number"); + if (PrestoBugs.bug23324) { + errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + } + errors.add("Cannot cast java.lang.String to java.util.List"); + errors.add("Unexpected subquery expression in logical plan"); + if (PrestoBugs.bugVerifyError) { + errors.add("VerifyError"); + } + if (PrestoBugs.bugCompilerFailed) { + errors.add("Compiler failed"); + errors.add("Error processing class definition"); + } + + // 9223372036854775808 + errors.add("Invalid numeric literal"); + + errors.add("Division by zero"); + errors.add("/ by zero"); + + errors.add("Cannot subtract hour, minutes or seconds from a date"); + errors.add("Cannot add hour, minutes or seconds to a date"); + + errors.add("DECIMAL scale must be in range"); + errors.add("IN value and list items must be the same type"); + errors.add("is not a valid timestamp literal"); + errors.add("Unknown time-zone ID"); + errors.add("GROUP BY position"); + + // ARRAY + errors.add("Unknown type: ARRAY"); + + // SELECT + errors.add("WHERE clause must evaluate to a boolean"); + errors.add("HAVING clause must evaluate to a boolean"); + errors.add("not yet implemented"); + + errors.add("Value expression and result of subquery must be of the same type for quantified comparison"); + errors.add("All IN list values must be the same type"); + errors.add("All CASE results must be the same type"); + errors.add("Mismatched types"); + errors.add("CASE operand type does not match WHEN clause operand type"); + errors.add("Subquery result type must be orderable"); + errors.add("Escape character must be followed by '%', '_' or the escape character itself"); + errors.add("Types are not comparable with NULLIF"); + errors.add("not of the same type"); + + if (PrestoBugs.bug23613) { + errors.add("at index 1"); + } + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + private static List getRegexErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("missing ]"); + errors.add("missing )"); + errors.add("invalid escape sequence"); + errors.add("no argument for repetition operator: "); + errors.add("bad repetition operator"); + errors.add("trailing \\"); + errors.add("invalid perl operator"); + errors.add("invalid character class range"); + errors.add("width is not integer"); + + return errors; + } + + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("SUBSTRING cannot handle negative lengths"); + errors.add("is undefined outside [-1,1]"); // ACOS etc + errors.add("invalid type specifier"); // PRINTF + errors.add("argument index out of range"); // PRINTF + errors.add("invalid format string"); // PRINTF + errors.add("number is too big"); // PRINTF + errors.add("Like pattern must not end with escape character!"); // LIKE + errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part + errors.add("extract specifier"); // date_part + errors.add("not recognized"); // date_part + errors.add("not supported"); // date_part + errors.add("Failed to cast"); + errors.add("Conversion Error"); + errors.add("Could not cast value"); + errors.add("Insufficient padding in RPAD"); // RPAD + errors.add("Could not choose a best candidate function for the function call"); // monthname + errors.add("expected a numeric precision field"); // ROUND + errors.add("with non-constant precision is not supported"); // ROUND + errors.add("Unexpected parameters"); + errors.add("not registered"); + errors.add("Expected: least(E) E:orderable"); + errors.add("Expected: greatest(E) E:orderable"); + errors.add("Expected: max_by(V, K) K:orderable, V, max_by(V, K, bigint) V, K:orderable"); + errors.add("Expected: min_by(V, K) K:orderable, V, min_by(V, K, bigint) V, K:orderable"); + return errors; + } + + // TODO: cover presto error + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getRegexErrors()); + errors.addAll(getExpressionErrors()); + + errors.add("NOT NULL constraint failed"); + errors.add("PRIMARY KEY or UNIQUE constraint violated"); + errors.add("duplicate key"); + errors.add("can't be cast because the value is out of range for the destination type"); + errors.add("Could not convert string"); + errors.add("Unimplemented type for cast"); + errors.add("field value out of range"); + errors.add("CHECK constraint failed"); + errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid + errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if + // the table has a primary key + errors.add("Could not cast value"); + errors.add("create unique index, table contains duplicate data"); + errors.add("Failed to cast"); + + errors.add("Values rows have mismatched types"); + errors.add("Mismatch at column"); + errors.add("This connector does not support updates or deletes"); + errors.add("Values rows have mismatched types"); + errors.add("Invalid numeric literal"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("must be an aggregate expression or appear in GROUP BY clause"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); + } + +} diff --git a/src/sqlancer/presto/PrestoGlobalState.java b/src/sqlancer/presto/PrestoGlobalState.java new file mode 100644 index 000000000..eb053712e --- /dev/null +++ b/src/sqlancer/presto/PrestoGlobalState.java @@ -0,0 +1,13 @@ +package sqlancer.presto; + +import java.sql.SQLException; + +import sqlancer.SQLGlobalState; + +public class PrestoGlobalState extends SQLGlobalState { + + @Override + protected PrestoSchema readSchema() throws SQLException { + return PrestoSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/presto/PrestoOptions.java b/src/sqlancer/presto/PrestoOptions.java new file mode 100644 index 000000000..f0c557ca5 --- /dev/null +++ b/src/sqlancer/presto/PrestoOptions.java @@ -0,0 +1,102 @@ +package sqlancer.presto; + +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(commandDescription = "Presto") +public class PrestoOptions implements DBMSSpecificOptions { + + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 8080; + + @Parameter(names = "--test-collate", arity = 1) + public boolean testCollate = true; + + @Parameter(names = "--test-check", description = "Allow generating CHECK constraints in tables", arity = 1) + public boolean testCheckConstraints = true; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions = true; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions (FALSE by default : Presto null handling in BETWEEN operator : https://prestodb.io/docs/current/functions/comparison.html )", arity = 1) + public boolean testBetween; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-time-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testTimeConstants = true; + + @Parameter(names = "--test-timestamp-constants", description = "Allow generating TIMESTAMP constants", arity = 1) + public boolean testTimestampConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--test-indexes", description = "Allow explicit (i.e. CREATE INDEX) and implicit (i.e., UNIQUE and PRIMARY KEY) indexes", arity = 1) + public boolean testIndexes = true; + + @Parameter(names = "--test-rowid", description = "Test tables' rowid columns", arity = 1) + public boolean testRowid = true; + + @Parameter(names = "--max-num-views", description = "The maximum number of views that can be generated for a database", arity = 1) + public int maxNumViews = 1; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--oracle") + public List oracles = List.of(PrestoOracleFactory.NOREC); + + @Parameter(names = "--catalog") + public String catalog = "memory"; + + @Parameter(names = "--schema") + public String schema = "test"; + + @Parameter(names = "--typed-generator", description = "the expression generator type - typed and untyped ") + public boolean typedGenerator = true; + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/presto/PrestoOracleFactory.java b/src/sqlancer/presto/PrestoOracleFactory.java new file mode 100644 index 000000000..3076357f2 --- /dev/null +++ b/src/sqlancer/presto/PrestoOracleFactory.java @@ -0,0 +1,73 @@ +package sqlancer.presto; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; +import sqlancer.presto.test.PrestoQueryPartitioningAggregateTester; +import sqlancer.presto.test.PrestoQueryPartitioningDistinctTester; +import sqlancer.presto.test.PrestoQueryPartitioningGroupByTester; +import sqlancer.presto.test.PrestoQueryPartitioningHavingTester; +import sqlancer.presto.test.PrestoQueryPartitioningWhereTester; + +public enum PrestoOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PrestoGlobalState globalState) { + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PrestoGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/presto/PrestoProvider.java b/src/sqlancer/presto/PrestoProvider.java new file mode 100644 index 000000000..be3ef4325 --- /dev/null +++ b/src/sqlancer/presto/PrestoProvider.java @@ -0,0 +1,195 @@ +package sqlancer.presto; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.presto.gen.PrestoInsertGenerator; +import sqlancer.presto.gen.PrestoTableGenerator; + +@AutoService(DatabaseProvider.class) +public class PrestoProvider extends SQLProviderAdapter { + + public PrestoProvider() { + super(PrestoGlobalState.class, PrestoOptions.class); + } + + // TODO : check actions based on connector + // returns number of actions + private static int mapActions(PrestoGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + if (Objects.requireNonNull(a) == Action.INSERT) { + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + // case UPDATE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + // case EXPLAIN: + // return r.getInteger(0, 2); + // case DELETE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + // case CREATE_VIEW: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + } + throw new AssertionError(a); + } + + @Override + public void generateDatabase(PrestoGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new PrestoTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + PrestoProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(PrestoGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + boolean useSSl = true; + if (globalState.getOptions().isDefaultUsername() && globalState.getOptions().isDefaultPassword()) { + username = "presto"; + password = null; + useSSl = false; + } + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = PrestoOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = PrestoOptions.DEFAULT_PORT; + } + String catalogName = globalState.getDbmsSpecificOptions().catalog; + String databaseName = globalState.getDatabaseName(); + String url = String.format("jdbc:presto://%s:%d/%s?SSL=%b", host, port, catalogName, useSSl); + Connection con = DriverManager.getConnection(url, username, password); + List schemaNames = getSchemaNames(con, catalogName, databaseName); + dropExistingTables(con, catalogName, databaseName, schemaNames); + dropSchema(globalState, con, catalogName, databaseName); + createSchema(globalState, con, catalogName, databaseName); + useSchema(globalState, con, catalogName, databaseName); + return new SQLConnection(con); + + } + + private static void useSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("USE " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("USE " + catalogName + "." + databaseName); + } + } + + private static void createSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("CREATE SCHEMA IF NOT EXISTS " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("CREATE SCHEMA IF NOT EXISTS " + catalogName + "." + databaseName); + } + } + + private static void dropSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("DROP SCHEMA IF EXISTS " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP SCHEMA IF EXISTS " + catalogName + "." + databaseName); + } + } + + private static List getSchemaNames(Connection con, String catalogName, String databaseName) + throws SQLException { + List schemaNames = new ArrayList<>(); + final String showSchemasSql = "SHOW SCHEMAS FROM " + catalogName + " LIKE '" + databaseName + "'"; + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(showSchemasSql)) { + while (rs.next()) { + schemaNames.add(rs.getString("Schema")); + } + } + } + return schemaNames; + } + + private static void dropExistingTables(Connection con, String catalogName, String databaseName, + List schemaNames) throws SQLException { + if (!schemaNames.isEmpty()) { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES FROM " + catalogName + "." + databaseName)) { + while (rs.next()) { + tableNames.add(rs.getString("Table")); + } + } + } + try (Statement s = con.createStatement()) { + for (String tableName : tableNames) { + s.execute("DROP TABLE IF EXISTS " + catalogName + "." + databaseName + "." + tableName); + } + } + } + } + + @Override + public String getDBMSName() { + return "presto"; + } + + public enum Action implements AbstractAction { + // SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES", new ExpectedErrors(), false, false)), // + INSERT(PrestoInsertGenerator::getQuery); + // TODO : check actions based on connector + // DELETE(PrestoDeleteGenerator::generate), // + // UPDATE(PrestoUpdateGenerator::getQuery), // + // CREATE_VIEW(PrestoViewGenerator::generate), // + // EXPLAIN((g) -> { + // ExpectedErrors errors = new ExpectedErrors(); + // PrestoErrors.addExpressionErrors(errors); + // PrestoErrors.addGroupByErrors(errors); + // return new SQLQueryAdapter( + // "EXPLAIN " + PrestoToStringVisitor + // .asString(PrestoRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + // errors); + // }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(PrestoGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + +} diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java new file mode 100644 index 000000000..2e668969d --- /dev/null +++ b/src/sqlancer/presto/PrestoSchema.java @@ -0,0 +1,487 @@ +package sqlancer.presto; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; + +public class PrestoSchema extends AbstractSchema { + + public PrestoSchema(List databaseTables) { + super(databaseTables); + } + + public static PrestoSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, databaseName, tableName); + boolean isView = matchesViewName(tableName); + PrestoTable t = new PrestoTable(tableName, databaseColumns, isView); + for (PrestoColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new PrestoSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + // TODO: UPDATE + // SHOW TABLES [ FROM schema ] [ LIKE pattern [ ESCAPE 'escape_character' ] ] + try (ResultSet rs = s.executeQuery("SHOW TABLES")) { + while (rs.next()) { + tableNames.add(rs.getString("Table")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("select " + " table_catalog " + " , table_schema " + + " , table_name " + " , column_name " + " , is_nullable " + " , data_type " + + " from information_schema.columns " + " where table_schema = '%s' and table_name = '%s'", + databaseName, tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + boolean isNullable = rs.getString("is_nullable").contentEquals("YES"); + PrestoColumn c = new PrestoColumn(columnName, getColumnType(dataType), false, isNullable); + columns.add(c); + } + } + } + + return columns; + } + + private static PrestoCompositeDataType getColumnType(String typeString) { + int bracesStart = typeString.indexOf('('); + String type; + int size = 0; + int precision = 0; + if (bracesStart != -1) { + type = typeString.substring(0, bracesStart); + } else { + type = typeString; + } + type = type.toUpperCase(); + + PrestoDataType primitiveType; + switch (type) { + case "INTEGER": + primitiveType = PrestoDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = PrestoDataType.INT; + size = 2; + break; + case "BIGINT": + primitiveType = PrestoDataType.INT; + size = 8; + break; + case "TINYINT": + primitiveType = PrestoDataType.INT; + size = 1; + break; + case "VARCHAR": + primitiveType = PrestoDataType.VARCHAR; + break; + case "VARBINARY": + primitiveType = PrestoDataType.VARBINARY; + break; + case "CHAR": + primitiveType = PrestoDataType.CHAR; + break; + case "FLOAT": + case "REAL": + primitiveType = PrestoDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + primitiveType = PrestoDataType.FLOAT; + size = 8; + break; + case "DECIMAL": + primitiveType = PrestoDataType.DECIMAL; + break; + case "BOOLEAN": + primitiveType = PrestoDataType.BOOLEAN; + break; + case "DATE": + primitiveType = PrestoDataType.DATE; + break; + case "TIME": + primitiveType = PrestoDataType.TIME; + break; + case "TIME WITH TIME ZONE": + primitiveType = PrestoDataType.TIME_WITH_TIME_ZONE; + break; + case "TIMESTAMP": + primitiveType = PrestoDataType.TIMESTAMP; + break; + case "TIMESTAMP WITH TIME ZONE": + primitiveType = PrestoDataType.TIMESTAMP_WITH_TIME_ZONE; + break; + case "INTERVAL DAY TO SECOND": + primitiveType = PrestoDataType.INTERVAL_DAY_TO_SECOND; + break; + case "INTERVAL YEAR TO MONTH": + primitiveType = PrestoDataType.INTERVAL_YEAR_TO_MONTH; + break; + case "JSON": + primitiveType = PrestoDataType.JSON; + break; + case "ARRAY": + int bracesEnd = typeString.length() - 1; + primitiveType = PrestoDataType.ARRAY; + PrestoCompositeDataType elementType = getColumnType(typeString.substring(bracesStart + 1, bracesEnd)); + return new PrestoCompositeDataType(primitiveType, elementType); + case "NULL": + primitiveType = PrestoDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + return new PrestoCompositeDataType(primitiveType, size, precision); + } + + public PrestoTables getRandomTableNonEmptyTables() { + return new PrestoTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public enum PrestoDataType { + BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, JSON, DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, + TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND, ARRAY, + // MAP, + // ROW, + // IPADDRESS, + // UID, + // IPPREFIX, + // HyperLogLog, + // P4HyperLogLog, + // KHyperLogLog, + // QDigest, + // TDigest, + NULL; + + public static PrestoDataType getRandomWithoutNull() { + PrestoDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == PrestoDataType.NULL); + return dt; + } + + public static List getNumericTypes() { + return Arrays.asList(INT, FLOAT, DECIMAL, DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, + TIMESTAMP_WITH_TIME_ZONE); + } + + public static List getComparableTypes() { + return Arrays.asList(BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, JSON, DATE, TIME, TIMESTAMP, + TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND); + } + + public static List getOrderableTypes() { + return Arrays.asList(BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, + // JSON, + DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, + INTERVAL_DAY_TO_SECOND, ARRAY); + } + + public static List getNumberTypes() { + return Arrays.asList(INT, FLOAT, DECIMAL); + } + + public static List getTemporalTypes() { + return Arrays.asList(DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE); + } + + public static List getIntervalTypes() { + return Arrays.asList(INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND); + } + + public static List getTextTypes() { + return Arrays.asList(VARCHAR, CHAR, VARBINARY, JSON); + } + + public boolean isNumeric() { + switch (this) { + case INT: + case FLOAT: + case DECIMAL: + return true; + default: + return false; + } + } + + public boolean isOrderable() { + return getOrderableTypes().contains(this); + } + + public PrestoCompositeDataType get() { + return PrestoCompositeDataType.fromDataType(this); + } + } + + public static class PrestoCompositeDataType { + + private final PrestoDataType dataType; + + private final int size; + + private final int scale; + + private final PrestoCompositeDataType elementType; + + public PrestoCompositeDataType(PrestoDataType dataType, int dataSize, int dataScale) { + this.dataType = dataType; + this.size = dataSize; + this.scale = dataScale; + this.elementType = null; + } + + public PrestoCompositeDataType(PrestoDataType dataType, PrestoCompositeDataType elementType) { + if (dataType != PrestoDataType.ARRAY) { + throw new IllegalArgumentException(); + } + this.dataType = dataType; + this.size = -1; + this.scale = -1; + this.elementType = elementType; + } + + public static PrestoCompositeDataType getRandomWithoutNull() { + PrestoDataType type = PrestoDataType.getRandomWithoutNull(); + int size; + int scale = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case DECIMAL: + size = Math.toIntExact(8); + scale = Math.toIntExact(4); + break; + case VARBINARY: + case JSON: + case VARCHAR: + case CHAR: + size = Math.toIntExact(Randomly.getNotCachedInteger(10, 250)); + break; + case ARRAY: + return new PrestoCompositeDataType(type, PrestoCompositeDataType.getRandomWithoutNull()); + case BOOLEAN: + case DATE: + case TIME: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP: + case TIMESTAMP_WITH_TIME_ZONE: + case INTERVAL_DAY_TO_SECOND: + case INTERVAL_YEAR_TO_MONTH: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new PrestoCompositeDataType(type, size, scale); + } + + public static PrestoCompositeDataType fromDataType(PrestoDataType type) { + int size; + int scale = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case DECIMAL: + size = Math.toIntExact(8); + scale = Math.toIntExact(4); + break; + case JSON: + case VARCHAR: + case CHAR: + size = Math.toIntExact(Randomly.getNotCachedInteger(10, 250)); + break; + case ARRAY: + return new PrestoCompositeDataType(type, PrestoCompositeDataType.getRandomWithoutNull()); + case BOOLEAN: + case VARBINARY: + case DATE: + case TIME: + case TIMESTAMP: + case TIMESTAMP_WITH_TIME_ZONE: + case TIME_WITH_TIME_ZONE: + case INTERVAL_DAY_TO_SECOND: + case INTERVAL_YEAR_TO_MONTH: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new PrestoCompositeDataType(type, size, scale); + } + + public PrestoDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public int getScale() { + if (scale == -1) { + throw new AssertionError(this); + } + return scale; + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return "BIGINT"; + case 4: + return "INTEGER"; + case 2: + return "SMALLINT"; + case 1: + return "TINYINT"; + default: + throw new AssertionError(size); + } + case VARBINARY: + return "VARBINARY"; + case JSON: + return "JSON"; + case VARCHAR: + return "VARCHAR" + "(" + size + ")"; + case CHAR: + return "CHAR" + "(" + size + ")"; + case FLOAT: + switch (size) { + case 4: + return "REAL"; + case 8: + return "DOUBLE"; + default: + throw new AssertionError(size); + } + case DECIMAL: + return "DECIMAL" + "(" + size + ", " + scale + ")"; + case BOOLEAN: + return "BOOLEAN"; + case TIMESTAMP_WITH_TIME_ZONE: + return "TIMESTAMP WITH TIME ZONE"; + case TIMESTAMP: + return "TIMESTAMP"; + case INTERVAL_YEAR_TO_MONTH: + return "INTERVAL YEAR TO MONTH"; + case INTERVAL_DAY_TO_SECOND: + return "INTERVAL DAY TO SECOND"; + case DATE: + return "DATE"; + case TIME: + return "TIME"; + case TIME_WITH_TIME_ZONE: + return "TIME WITH TIME ZONE"; + case ARRAY: + return "ARRAY(" + elementType + ")"; + case NULL: + return "NULL"; + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + public PrestoCompositeDataType getElementType() { + return elementType; + } + + public boolean isOrderable() { + if (dataType == PrestoDataType.ARRAY) { + assert elementType != null; + return elementType.isOrderable(); + } + return dataType.isOrderable(); + } + + } + + public static class PrestoColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public PrestoColumn(String name, PrestoCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + @Override + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + public boolean isOrderable() { + return getType().getPrimitiveDataType().isOrderable(); + } + + } + + public static class PrestoTables extends AbstractTables { + + public PrestoTables(List tables) { + super(tables); + } + + } + + public static class PrestoTable extends AbstractRelationalTable { + + public PrestoTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + +} diff --git a/src/sqlancer/presto/PrestoToStringVisitor.java b/src/sqlancer/presto/PrestoToStringVisitor.java new file mode 100644 index 000000000..11c6d5fa6 --- /dev/null +++ b/src/sqlancer/presto/PrestoToStringVisitor.java @@ -0,0 +1,148 @@ +package sqlancer.presto; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.presto.ast.PrestoAtTimeZoneOperator; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionWithoutParenthesis; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoMultiValuedComparison; +import sqlancer.presto.ast.PrestoQuantifiedComparison; +import sqlancer.presto.ast.PrestoSelect; + +public class PrestoToStringVisitor extends NewToStringVisitor { + + public static String asString(PrestoExpression expr) { + PrestoToStringVisitor visitor = new PrestoToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + @Override + public void visitSpecific(PrestoExpression expr) { + if (expr instanceof PrestoConstant) { + visit((PrestoConstant) expr); + } else if (expr instanceof PrestoSelect) { + visit((PrestoSelect) expr); + } else if (expr instanceof PrestoJoin) { + visit((PrestoJoin) expr); + } else if (expr instanceof PrestoCastFunction) { + visit((PrestoCastFunction) expr); + } else if (expr instanceof PrestoFunctionWithoutParenthesis) { + visit((PrestoFunctionWithoutParenthesis) expr); + } else if (expr instanceof PrestoAtTimeZoneOperator) { + visit((PrestoAtTimeZoneOperator) expr); + } else if (expr instanceof PrestoMultiValuedComparison) { + visit((PrestoMultiValuedComparison) expr); + } else if (expr instanceof PrestoQuantifiedComparison) { + visit((PrestoQuantifiedComparison) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(PrestoJoin join) { + visit((PrestoExpression) join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit((PrestoExpression) join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(PrestoConstant constant) { + sb.append(constant.toString()); + } + + private void visit(PrestoAtTimeZoneOperator timeZoneOperator) { + visit(timeZoneOperator.getExpr()); + sb.append(" AT TIME ZONE "); + sb.append(timeZoneOperator.getTimeZone()); + } + + private void visit(PrestoFunctionWithoutParenthesis prestoFunctionWithoutParenthesis) { + sb.append(prestoFunctionWithoutParenthesis.getExpr()); + } + + private void visit(PrestoSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public void visit(PrestoCastFunction cast) { + sb.append("CAST(("); + visit(cast.getExpr()); + sb.append(") AS "); + sb.append(cast.getType().toString()); + sb.append(")"); + } + + public void visit(PrestoMultiValuedComparison comp) { + sb.append("("); + visit(comp.getLeft()); + sb.append(" "); + sb.append(comp.getOp().getStringRepresentation()); + sb.append(" "); + sb.append(comp.getType()); + sb.append(" (VALUES "); + visit(comp.getRight()); + sb.append(")"); + sb.append(")"); + } + + public void visit(PrestoQuantifiedComparison comp) { + sb.append("("); + visit(comp.getLeft()); + sb.append(" "); + sb.append(comp.getOp().getStringRepresentation()); + sb.append(" "); + sb.append(comp.getType()); + sb.append(" ( "); + visit(comp.getRight()); + sb.append(" ) "); + sb.append(")"); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAggregateFunction.java b/src/sqlancer/presto/ast/PrestoAggregateFunction.java new file mode 100644 index 000000000..25840933a --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAggregateFunction.java @@ -0,0 +1,713 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public enum PrestoAggregateFunction implements PrestoFunction { + + // General Aggregate Functions + + // arbitrary(x) → [same as input] + // Returns an arbitrary non-null value of x, if one exists. + ARBITRARY("arbitrary", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return PrestoDataType.getRandomWithoutNull(); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // array_agg(x) → array<[same as input]># + // Returns an array created from the input x elements. + + // avg(x) → double + // Returns the average (arithmetic mean) of all input values. + AVG("avg", PrestoDataType.FLOAT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { + Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) }; + } + }, + // avg(time interval type) → time interval type# + // Returns the average interval length of all input values. + AVG_INTERVAL_YM("avg", PrestoDataType.INTERVAL_YEAR_TO_MONTH, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + AVG_INTERVAL_DS("avg", PrestoDataType.INTERVAL_DAY_TO_SECOND, PrestoDataType.INTERVAL_DAY_TO_SECOND), + + // bool_and(boolean) → boolean# + // Returns TRUE if every input value is TRUE, otherwise FALSE. + BOOL_AND("bool_and", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // bool_or(boolean) → boolean# + // Returns TRUE if any input value is TRUE, otherwise FALSE. + BOOL_OR("bool_or", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // checksum(x) → varbinary# + // Returns an order-insensitive checksum of the given values. + CHECKSUM("checksum", PrestoDataType.VARBINARY) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getComparableTypes()) }; + } + }, + + // count(*) → bigint# + // Returns the number of input rows. + COUNT_ALL("count(*)", PrestoDataType.INT), + // count(x) → bigint# + // Returns the number of non-null input values. + COUNT_NOARGS("count", PrestoDataType.INT), COUNT("count", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.getRandomWithoutNull()) }; + } + }, + // count_if(x) → bigint# + // Returns the number of TRUE input values. This function is equivalent to count(CASE WHEN x THEN 1 END). + COUNT_IF("count_if", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.getRandomWithoutNull()) }; + } + }, + // every(boolean) → boolean# + // This is an alias for bool_and(). + EVERY("every", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // geometric_mean(x) → double# + // Returns the geometric mean of all input values. + GEOMETRIC_MEAN("geometric_mean", PrestoDataType.FLOAT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { + Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) }; + } + }, + // max_by(x, y) → [same as x]# + // Returns the value of x associated with the maximum value of y over all input values. + MAX_BY("max_by", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), + Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // max_by(x, y, n) → array<[same as x]># + // Returns n values of x associated with the n largest of all input values of y in descending order of y. + + // min_by(x, y) → [same as x]# + // Returns the value of x associated with the minimum value of y over all input values. + MIN_BY("min_by", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), + Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + // TODO: + // + // min_by(x, y, n) → array<[same as x]> + // Returns n values of x associated with the n smallest of all input values of y in ascending order of y. + + // max(x) → [same as input] + // Returns the maximum value of all input values. + MAX("max", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + boolean isCompatible = PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY && returnType.toString().contains("JSON")) { + isCompatible = false; + } + return isCompatible; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public PrestoCompositeDataType getCompositeReturnType() { + PrestoDataType dataType = Randomly.fromList(PrestoDataType.getOrderableTypes()); + PrestoCompositeDataType returnType; + do { + returnType = PrestoCompositeDataType.fromDataType(dataType); + } while (!isCompatibleWithReturnType(returnType)); + return returnType; + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, true); + } + + }, + + // TODO: + // + // max(x, n) → array<[same as x]># + // Returns n largest values of all input values of x. + + // min(x) → [same as input]# + // Returns the minimum value of all input values. + MIN("min", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + boolean orderable = PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY && returnType.toString().contains("JSON")) { + orderable = false; + } + return orderable; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // min(x, n) → array<[same as x]># + // Returns n smallest values of all input values of x. + + // TODO: + // + // reduce_agg(inputValue T, initialState S, inputFunction(S, T, S), combineFunction(S, S, S)) → S# + // Reduces all input values into a single value. inputFunction will be invoked for each input value. In addition to + // taking the input value, inputFunction takes the current state, initially initialState, and returns the new state. + // combineFunction will be invoked to combine two states into a new state. The final state is returned: + // + // SELECT id, reduce_agg(value, (a, b) -> a + b, (a, b) -> a + b) + // FROM ( + // VALUES + // (1, 2), + // (1, 3), + // (1, 4), + // (2, 20), + // (2, 30), + // (2, 40) + // ) AS t(id, value) + // GROUP BY id; + // -- (1, 9) + // -- (2, 90) + // + // SELECT id, reduce_agg(value, (a, b) -> a * b, (a, b) -> a * b) + // FROM ( + // VALUES + // (1, 2), + // (1, 3), + // (1, 4), + // (2, 20), + // (2, 30), + // (2, 40) + // ) AS t(id, value) + // GROUP BY id; + // -- (1, 24) + // -- (2, 24000) + // The state type must be a boolean, integer, floating-point, or date/time/interval. + + // TODO: + // + // set_agg(x) → array<[same as input]># + // Returns an array created from the distinct input x elements. + + // TODO: + // + // set_union(array(T)) -> array(T)# + // Returns an array of all the distinct values contained in each array of the input + // + // Example: + // + // SELECT set_union(elements) + // FROM ( + // VALUES + // ARRAY[1, 3], + // ARRAY[2, 4] + // ) AS t(elements); + // Returns ARRAY[1, 3, 4] + + // sum(x) → [same as input]# + // Returns the sum of all input values. + SUM("sum", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) + .contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + // sum(time interval type) → time interval type# + // Returns the average interval length of all input values. + SUM_INTERVAL_YM("sum", PrestoDataType.INTERVAL_YEAR_TO_MONTH, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + SUM_INTERVAL_DS("sum", PrestoDataType.INTERVAL_DAY_TO_SECOND, PrestoDataType.INTERVAL_DAY_TO_SECOND), + + // Bitwise Aggregate Functions# + + // bitwise_and_agg(x) → bigint# + // Returns the bitwise AND of all input values in 2’s complement representation. + BITWISE_AND_AGG("bitwise_and_agg", PrestoDataType.INT, PrestoDataType.INT), + + // bitwise_or_agg(x) → bigint# + // Returns the bitwise OR of all input values in 2’s complement representation. + BITWISE_OR_AGG("bitwise_or_agg", PrestoDataType.INT, PrestoDataType.INT), + + // TODO: + // + // Map Aggregate Functions + + // histogram(x)# + // Returns a map containing the count of the number of times each input value occurs. + // + // map_agg(key, value)# + // Returns a map created from the input key / value pairs. + // + // map_union(x(K, V)) -> map(K, V)# + // Returns the union of all the input maps. If a key is found in multiple input maps, that key’s value in the + // resulting map comes from an arbitrary input map. + // + // map_union_sum(x(K, V)) -> map(K, V)# + // Returns the union of all the input maps summing the values of matching keys in all the maps. All null values in + // the original maps are coalesced to 0. + // + // multimap_agg(key, value)# + // Returns a multimap created from the input key / value pairs. Each key can be associated with multiple values. + + // Approximate Aggregate Functions# + // approx_distinct(x) → bigint# + // Returns the approximate number of distinct input values. This function provides an approximation of + // count(DISTINCT x). + // Zero is returned if all input values are null. + // This function should produce a standard error of 2.3%, which is the standard deviation of the (approximately + // normal) + // error distribution over all possible sets. It does not guarantee an upper bound on the error for any specific + // input set. + APPROX_DISTINCT("approx_distinct", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + }, + // + // approx_distinct(x, e) → bigint# + // Returns the approximate number of distinct input values. This function provides an approximation of + // count(DISTINCT x). Zero is returned if all input values are null. + // + // This function should produce a standard error of no more than e, which is the standard deviation of the + // (approximately normal) error distribution over all possible sets. It does not guarantee an upper bound on the + // error for any specific input set. The current implementation of this function requires that e be in the range of + // [0.0040625, 0.26000]. + APPROX_DISTINCT_2("approx_distinct", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getOrderableTypes()), PrestoDataType.FLOAT }; + } + }, + // approx_percentile(x, percentage) → [same as x]# + // Returns the approximate percentile for all input values of x at the given percentage. + // The value of percentage must be between zero and one and must be constant for all input rows. + APPROX_PERCENTILE("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.FLOAT }; + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // approx_percentile(x, percentage, accuracy) → [same as x]# + // As approx_percentile(x, percentage), but with a maximum rank error of accuracy. + // The value of accuracy must be between zero and one (exclusive) and must be constant for all input rows. + // Note that a lower “accuracy” is really a lower error threshold, and thus more accurate. The default accuracy is + // 0.01. + APPROX_PERCENTILE_ACCURACY("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.FLOAT, PrestoDataType.FLOAT }; + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoFloatConstant(0.01D)); + } else { + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // TODO: + // + // approx_percentile(x, percentages) → array<[same as x]># + // Returns the approximate percentile for all input values of x at each of the specified percentages. Each element + // of the percentages array must be between zero and one, and the array must be constant for all input rows. + // + // approx_percentile(x, percentages, accuracy) → array<[same as x]># + // As approx_percentile(x, percentages), but with a maximum rank error of accuracy. + + // approx_percentile(x, w, percentage) → [same as x]# + // Returns the approximate weighed percentile for all input values of x using the per-item weight w at the + // percentage p. + // The weight must be an integer value of at least one. + // It is effectively a replication count for the value x in the percentile set. + // The value of p must be between zero and one and must be constant for all input rows. + APPROX_PERCENTILE_WEIGHT("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.INT, PrestoDataType.FLOAT }; + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoIntConstant(1)); + } else { + arguments.add(new PrestoConstant.PrestoIntConstant(Randomly.smallNumber())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // approx_percentile(x, w, percentage, accuracy) → [same as x]# + // As approx_percentile(x, w, percentage), but with a maximum rank error of accuracy. + APPROX_PERCENTILE_PERCENTAGE_ACCURACY("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.FLOAT }; + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoIntConstant(1)); + } else { + arguments.add(new PrestoConstant.PrestoIntConstant(Randomly.smallNumber())); + } + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoFloatConstant(0.01D)); + } else { + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }; + + // TODO: + // + // approx_percentile(x, w, percentages) → array<[same as x]># + // Returns the approximate weighed percentile for all input values of x using the per-item weight w at each of the + // given percentages specified in the array. The weight must be an integer value of at least one. It is effectively + // a replication count for the value x in the percentile set. Each element of the array must be between zero and + // one, and the array must be constant for all input rows. + // + // approx_percentile(x, w, percentages, accuracy) → array<[same as x]># + // As approx_percentile(x, w, percentages), but with a maximum rank error of accuracy. + // + // approx_set(x) → HyperLogLog + // See HyperLogLog Functions. + // + // merge(x) → HyperLogLog + // See HyperLogLog Functions. + // + // khyperloglog_agg(x) → KHyperLogLog + // See KHyperLogLog Functions. + + // TODO: + // + // merge(qdigest(T)) -> qdigest(T) + // See Quantile Digest Functions. + // + // qdigest_agg(x) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // qdigest_agg(x, w) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // qdigest_agg(x, w, accuracy) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // numeric_histogram(buckets, value, weight) → map# + // Computes an approximate histogram with up to buckets number of buckets for all values with a per-item weight of + // weight. + // The keys of the returned map are roughly the center of the bin, and the entry is the total weight of the bin. + // The algorithm is based loosely on [BenHaimTomTov2010]. + // + // buckets must be a bigint. value and weight must be numeric. + // + // numeric_histogram(buckets, value) → map# + // Computes an approximate histogram with up to buckets number of buckets for all values. This function is + // equivalent to the variant of numeric_histogram() that takes a weight, with a per-item weight of 1. In this case, + // the total weight in the returned map is the count of items in the bin. + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoAggregateFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + } + + PrestoAggregateFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + public static PrestoAggregateFunction getRandomMetamorphicOracle() { + return Randomly.fromOptions(ARBITRARY, AVG, AVG_INTERVAL_YM, AVG_INTERVAL_DS, BOOL_AND, BOOL_OR, CHECKSUM, + COUNT_ALL, COUNT_NOARGS, COUNT, COUNT_IF, EVERY, GEOMETRIC_MEAN, MAX_BY, MIN_BY, MAX, MIN, SUM, + SUM_INTERVAL_YM, SUM_INTERVAL_DS, BITWISE_AND_AGG, BITWISE_OR_AGG); + } + + public static PrestoAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public static List getFunctionsCompatibleWith(PrestoCompositeDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)).collect(Collectors.toList()); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + + @Override + public int getNumberOfArguments() { + return 1; + } + + public List getReturnTypes(PrestoSchema.PrestoDataType dataType) { + return Collections.singletonList(dataType); + } + + public PrestoDataType getReturnType() { + if (returnType == null) { + return PrestoDataType.getRandomWithoutNull(); + } + return returnType; + } + + public PrestoCompositeDataType getCompositeReturnType() { + PrestoDataType dataType = getReturnType(); + return PrestoCompositeDataType.fromDataType(dataType); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAlias.java b/src/sqlancer/presto/ast/PrestoAlias.java new file mode 100644 index 000000000..b327bed11 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAlias.java @@ -0,0 +1,9 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class PrestoAlias extends NewAliasNode implements PrestoExpression { + public PrestoAlias(PrestoExpression expr, String alias) { + super(expr, alias); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java new file mode 100644 index 000000000..ac73a0a79 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java @@ -0,0 +1,20 @@ +package sqlancer.presto.ast; + +public class PrestoAtTimeZoneOperator implements PrestoExpression { + + private final PrestoExpression expr; + private final PrestoExpression timeZone; + + public PrestoAtTimeZoneOperator(PrestoExpression expr, PrestoExpression timeZone) { + this.expr = expr; + this.timeZone = timeZone; + } + + public PrestoExpression getExpr() { + return expr; + } + + public PrestoExpression getTimeZone() { + return timeZone; + } +} diff --git a/src/sqlancer/presto/ast/PrestoBetweenOperation.java b/src/sqlancer/presto/ast/PrestoBetweenOperation.java new file mode 100644 index 000000000..394a5f675 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class PrestoBetweenOperation extends NewBetweenOperatorNode implements PrestoExpression { + public PrestoBetweenOperation(PrestoExpression left, PrestoExpression middle, PrestoExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/presto/ast/PrestoBinaryOperation.java b/src/sqlancer/presto/ast/PrestoBinaryOperation.java new file mode 100644 index 000000000..90f83f31a --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class PrestoBinaryOperation extends NewBinaryOperatorNode implements PrestoExpression { + public PrestoBinaryOperation(PrestoExpression left, PrestoExpression right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/presto/ast/PrestoCaseOperation.java b/src/sqlancer/presto/ast/PrestoCaseOperation.java new file mode 100644 index 000000000..117c54bbb --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoCaseOperation.java @@ -0,0 +1,12 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class PrestoCaseOperation extends NewCaseOperatorNode implements PrestoExpression { + public PrestoCaseOperation(PrestoExpression switchCondition, List conditions, + List expressions, PrestoExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/presto/ast/PrestoCastFunction.java b/src/sqlancer/presto/ast/PrestoCastFunction.java new file mode 100644 index 000000000..14798267d --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoCastFunction.java @@ -0,0 +1,23 @@ +package sqlancer.presto.ast; + +import sqlancer.presto.PrestoSchema; + +public class PrestoCastFunction implements PrestoExpression { + + private final PrestoExpression expr; + private final PrestoSchema.PrestoCompositeDataType type; + + public PrestoCastFunction(PrestoExpression expr, PrestoSchema.PrestoCompositeDataType type) { + this.expr = expr; + this.type = type; + } + + public PrestoExpression getExpr() { + return expr; + } + + public PrestoSchema.PrestoCompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoColumnReference.java b/src/sqlancer/presto/ast/PrestoColumnReference.java new file mode 100644 index 000000000..8bc6db0ec --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoColumnReference.java @@ -0,0 +1,13 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.presto.PrestoSchema; + +public class PrestoColumnReference extends ColumnReferenceNode + implements PrestoExpression { + + public PrestoColumnReference(PrestoSchema.PrestoColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoComparisonFunction.java b/src/sqlancer/presto/ast/PrestoComparisonFunction.java new file mode 100644 index 000000000..1125c3e35 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoComparisonFunction.java @@ -0,0 +1,81 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoComparisonFunction implements PrestoFunction { + + // comparison + + // Returns the largest of the provided values. + // → [same as input] + GREATEST("greatest", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + }, + // Returns the smallest of the provided values. + // → [same as input]# + LEAST("least", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoComparisonFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoConditionalFunction.java b/src/sqlancer/presto/ast/PrestoConditionalFunction.java new file mode 100644 index 000000000..e603a7e5d --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoConditionalFunction.java @@ -0,0 +1,95 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoConditionalFunction implements PrestoFunction { + + // Conditional functions + IF_TRUE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { PrestoDataType.BOOLEAN, returnType.getPrimitiveDataType() }; + } + }, + + IF_TRUE_FALSE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { PrestoDataType.BOOLEAN, returnType.getPrimitiveDataType(), + returnType.getPrimitiveDataType() }; + } + }, + + NULLIF("nullif", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + COALESCE("coalesce", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + List prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private final PrestoDataType returnType; + private final String functionName; + + PrestoConditionalFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public int getNumberOfArguments() { + return getArgumentTypes(PrestoCompositeDataType.fromDataType(returnType)).length; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoConstant.java b/src/sqlancer/presto/ast/PrestoConstant.java new file mode 100644 index 000000000..a469b983c --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoConstant.java @@ -0,0 +1,808 @@ +package sqlancer.presto.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoConstantUtils; +import sqlancer.presto.PrestoSchema; + +public abstract class PrestoConstant implements PrestoExpression { + + private static final String[] TIME_ZONES = { "Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa", + "Africa/Algiers", "Africa/Asmara", "Africa/Asmera", "Africa/Bamako", "Africa/Bangui", "Africa/Banjul", + "Africa/Bissau", "Africa/Blantyre", "Africa/Brazzaville", "Africa/Bujumbura", "Africa/Cairo", + "Africa/Casablanca", "Africa/Ceuta", "Africa/Conakry", "Africa/Dakar", "Africa/Dar_es_Salaam", + "Africa/Djibouti", "Africa/Douala", "Africa/El_Aaiun", "Africa/Freetown", "Africa/Gaborone", + "Africa/Harare", "Africa/Johannesburg", "Africa/Juba", "Africa/Kampala", "Africa/Khartoum", "Africa/Kigali", + "Africa/Kinshasa", "Africa/Lagos", "Africa/Libreville", "Africa/Lome", "Africa/Luanda", "Africa/Lubumbashi", + "Africa/Lusaka", "Africa/Malabo", "Africa/Maputo", "Africa/Maseru", "Africa/Mbabane", "Africa/Mogadishu", + "Africa/Monrovia", "Africa/Nairobi", "Africa/Ndjamena", "Africa/Niamey", "Africa/Nouakchott", + "Africa/Ouagadougou", "Africa/Porto-Novo", "Africa/Sao_Tome", "Africa/Timbuktu", "Africa/Tripoli", + "Africa/Tunis", "Africa/Windhoek", "America/Adak", "America/Anchorage", "America/Anguilla", + "America/Antigua", "America/Araguaina", "America/Argentina/Buenos_Aires", "America/Argentina/Catamarca", + "America/Argentina/ComodRivadavia", "America/Argentina/Cordoba", "America/Argentina/Jujuy", + "America/Argentina/La_Rioja", "America/Argentina/Mendoza", "America/Argentina/Rio_Gallegos", + "America/Argentina/Salta", "America/Argentina/San_Juan", "America/Argentina/San_Luis", + "America/Argentina/Tucuman", "America/Argentina/Ushuaia", "America/Aruba", "America/Asuncion", + "America/Atikokan", "America/Atka", "America/Bahia", "America/Barbados", "America/Belem", "America/Belize", + "America/Blanc-Sablon", "America/Boa_Vista", "America/Bogota", "America/Boise", "America/Buenos_Aires", + "America/Cambridge_Bay", "America/Campo_Grande", "America/Cancun", "America/Caracas", "America/Catamarca", + "America/Cayenne", "America/Cayman", "America/Chicago", "America/Chihuahua", "America/Coral_Harbour", + "America/Cordoba", "America/Costa_Rica", "America/Creston", "America/Cuiaba", "America/Curacao", + "America/Danmarkshavn", "America/Dawson", "America/Dawson_Creek", "America/Denver", "America/Detroit", + "America/Dominica", "America/Edmonton", "America/Eirunepe", "America/El_Salvador", "America/Ensenada", + "America/Fort_Nelson", "America/Fort_Wayne", "America/Fortaleza", "America/Glace_Bay", "America/Godthab", + "America/Goose_Bay", "America/Grand_Turk", "America/Grenada", "America/Guadeloupe", "America/Guatemala", + "America/Guayaquil", "America/Guyana", "America/Halifax", "America/Havana", "America/Indiana/Indianapolis", + "America/Indiana/Knox", "America/Indiana/Marengo", "America/Indiana/Petersburg", + "America/Indiana/Tell_City", "America/Indiana/Vevay", "America/Indiana/Vincennes", + "America/Indiana/Winamac", "America/Indianapolis", "America/Inuvik", "America/Iqaluit", "America/Jamaica", + "America/Jujuy", "America/Juneau", "America/Kentucky/Louisville", "America/Kentucky/Monticello", + "America/Knox_IN", "America/Kralendijk", "America/La_Paz", "America/Lima", "America/Los_Angeles", + "America/Louisville", "America/Lower_Princes", "America/Maceio", "America/Managua", "America/Manaus", + "America/Marigot", "America/Martinique", "America/Matamoros", "America/Mendoza", "America/Menominee", + "America/Merida", "America/Metlakatla", "America/Mexico_City", "America/Miquelon", "America/Moncton", + "America/Monterrey", "America/Montevideo", "America/Montreal", "America/Montserrat", "America/Nassau", + "America/New_York", "America/Nipigon", "America/Nome", "America/Noronha", "America/North_Dakota/Beulah", + "America/North_Dakota/Center", "America/North_Dakota/New_Salem", "America/Nuuk", "America/Ojinaga", + "America/Panama", "America/Pangnirtung", "America/Paramaribo", "America/Phoenix", "America/Port-au-Prince", + "America/Port_of_Spain", "America/Porto_Acre", "America/Porto_Velho", "America/Puerto_Rico", + "America/Punta_Arenas", "America/Rainy_River", "America/Rankin_Inlet", "America/Recife", "America/Regina", + "America/Resolute", "America/Rio_Branco", "America/Rosario", "America/Santa_Isabel", "America/Santarem", + "America/Santiago", "America/Santo_Domingo", "America/Sao_Paulo", "America/Scoresbysund", + "America/Shiprock", "America/Sitka", "America/St_Barthelemy", "America/St_Johns", "America/St_Kitts", + "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", "America/Swift_Current", + "America/Tegucigalpa", "America/Thule", "America/Thunder_Bay", "America/Tijuana", "America/Toronto", + "America/Tortola", "America/Vancouver", "America/Virgin", "America/Whitehorse", "America/Winnipeg", + "America/Yakutat", "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", + "Antarctica/DumontDUrville", "Antarctica/Macquarie", "Antarctica/Mawson", "Antarctica/McMurdo", + "Antarctica/Palmer", "Antarctica/Rothera", "Antarctica/South_Pole", "Antarctica/Syowa", "Antarctica/Troll", + "Antarctica/Vostok", "Arctic/Longyearbyen", "Asia/Aden", "Asia/Almaty", "Asia/Amman", "Asia/Anadyr", + "Asia/Aqtau", "Asia/Aqtobe", "Asia/Ashgabat", "Asia/Ashkhabad", "Asia/Atyrau", "Asia/Baghdad", + "Asia/Bahrain", "Asia/Baku", "Asia/Bangkok", "Asia/Barnaul", "Asia/Beirut", "Asia/Bishkek", "Asia/Brunei", + "Asia/Calcutta", "Asia/Chita", "Asia/Choibalsan", "Asia/Chongqing", "Asia/Chungking", "Asia/Colombo", + "Asia/Dacca", "Asia/Dhaka", "Asia/Dili", "Asia/Dubai", "Asia/Dushanbe", "Asia/Famagusta", "Asia/Gaza", + "Asia/Harbin", "Asia/Hebron", "Asia/Ho_Chi_Minh", "Asia/Hong_Kong", "Asia/Hovd", "Asia/Irkutsk", + "Asia/Istanbul", "Asia/Jakarta", "Asia/Jayapura", "Asia/Jerusalem", "Asia/Kabul", "Asia/Kamchatka", + "Asia/Karachi", "Asia/Kashgar", "Asia/Kathmandu", "Asia/Katmandu", "Asia/Khandyga", "Asia/Kolkata", + "Asia/Krasnoyarsk", "Asia/Kuala_Lumpur", "Asia/Kuching", "Asia/Kuwait", "Asia/Macao", "Asia/Macau", + "Asia/Magadan", "Asia/Makassar", "Asia/Manila", "Asia/Muscat", "Asia/Nicosia", "Asia/Novokuznetsk", + "Asia/Novosibirsk", "Asia/Omsk", "Asia/Oral", "Asia/Phnom_Penh", "Asia/Pontianak", "Asia/Pyongyang", + "Asia/Qatar", "Asia/Qostanay", "Asia/Qyzylorda", "Asia/Rangoon", "Asia/Riyadh", "Asia/Saigon", + "Asia/Sakhalin", "Asia/Samarkand", "Asia/Seoul", "Asia/Shanghai", "Asia/Singapore", "Asia/Srednekolymsk", + "Asia/Taipei", "Asia/Tashkent", "Asia/Tbilisi", "Asia/Tehran", "Asia/Tel_Aviv", "Asia/Thimbu", + "Asia/Thimphu", "Asia/Tokyo", "Asia/Tomsk", "Asia/Ujung_Pandang", "Asia/Ulaanbaatar", "Asia/Ulan_Bator", + "Asia/Urumqi", "Asia/Ust-Nera", "Asia/Vientiane", "Asia/Vladivostok", "Asia/Yakutsk", "Asia/Yangon", + "Asia/Yekaterinburg", "Asia/Yerevan", "Atlantic/Azores", "Atlantic/Bermuda", "Atlantic/Canary", + "Atlantic/Cape_Verde", "Atlantic/Faeroe", "Atlantic/Faroe", "Atlantic/Jan_Mayen", "Atlantic/Madeira", + "Atlantic/Reykjavik", "Atlantic/South_Georgia", "Atlantic/St_Helena", "Atlantic/Stanley", "Australia/ACT", + "Australia/Adelaide", "Australia/Brisbane", "Australia/Broken_Hill", "Australia/Canberra", + "Australia/Currie", "Australia/Darwin", "Australia/Eucla", "Australia/Hobart", "Australia/LHI", + "Australia/Lindeman", "Australia/Lord_Howe", "Australia/Melbourne", "Australia/North", "Australia/Perth", + "Australia/Queensland", "Australia/South", "Australia/Sydney", "Australia/Tasmania", "Australia/Victoria", + "Australia/West", "Australia/Yancowinna", "Brazil/Acre", "Brazil/DeNoronha", "Brazil/East", "Brazil/West", + "CET", "CST6CDT", "Canada/Atlantic", "Canada/Central", "Canada/Eastern", "Canada/Mountain", + "Canada/Newfoundland", "Canada/Pacific", "Canada/Saskatchewan", "Canada/Yukon", "Chile/Continental", + "Chile/EasterIsland", "Cuba", "EET", "EST5EDT", "Egypt", "Eire", "Etc/GMT", "Etc/GMT+0", "Etc/GMT+1", + "Etc/GMT+10", "Etc/GMT+11", "Etc/GMT+12", "Etc/GMT+2", "Etc/GMT+3", "Etc/GMT+4", "Etc/GMT+5", "Etc/GMT+6", + "Etc/GMT+7", "Etc/GMT+8", "Etc/GMT+9", "Etc/GMT-0", "Etc/GMT-1", "Etc/GMT-10", "Etc/GMT-11", "Etc/GMT-12", + "Etc/GMT-13", "Etc/GMT-14", "Etc/GMT-2", "Etc/GMT-3", "Etc/GMT-4", "Etc/GMT-5", "Etc/GMT-6", "Etc/GMT-7", + "Etc/GMT-8", "Etc/GMT-9", "Etc/GMT0", "Etc/Greenwich", "Etc/UCT", "Etc/UTC", "Etc/Universal", "Etc/Zulu", + "Europe/Amsterdam", "Europe/Andorra", "Europe/Astrakhan", "Europe/Athens", "Europe/Belfast", + "Europe/Belgrade", "Europe/Berlin", "Europe/Bratislava", "Europe/Brussels", "Europe/Bucharest", + "Europe/Budapest", "Europe/Busingen", "Europe/Chisinau", "Europe/Copenhagen", "Europe/Dublin", + "Europe/Gibraltar", "Europe/Guernsey", "Europe/Helsinki", "Europe/Isle_of_Man", "Europe/Istanbul", + "Europe/Jersey", "Europe/Kaliningrad", "Europe/Kiev", "Europe/Kirov", "Europe/Lisbon", "Europe/Ljubljana", + "Europe/London", "Europe/Luxembourg", "Europe/Madrid", "Europe/Malta", "Europe/Mariehamn", "Europe/Minsk", + "Europe/Monaco", "Europe/Moscow", "Europe/Nicosia", "Europe/Oslo", "Europe/Paris", "Europe/Podgorica", + "Europe/Prague", "Europe/Riga", "Europe/Rome", "Europe/Samara", "Europe/San_Marino", "Europe/Sarajevo", + "Europe/Saratov", "Europe/Simferopol", "Europe/Skopje", "Europe/Sofia", "Europe/Stockholm", "Europe/Tirane", + "Europe/Tiraspol", "Europe/Ulyanovsk", "Europe/Uzhgorod", "Europe/Vaduz", "Europe/Vatican", "Europe/Vienna", + "Europe/Vilnius", "Europe/Volgograd", "Europe/Warsaw", "Europe/Zagreb", "Europe/Zaporozhye", + "Europe/Zurich", "GB", "GB-Eire", "GMT", "GMT0", "Greenwich", "Hongkong", "Iceland", "Indian/Antananarivo", + "Indian/Chagos", "Indian/Christmas", "Indian/Cocos", "Indian/Comoro", "Indian/Kerguelen", "Indian/Mahe", + "Indian/Maldives", "Indian/Mauritius", "Indian/Mayotte", "Indian/Reunion", "Iran", "Israel", "Jamaica", + "Japan", "Kwajalein", "Libya", "MET", "MST7MDT", "Mexico/General", "NZ", "NZ-CHAT", "Navajo", "PRC", + "PST8PDT", "Pacific/Apia", "Pacific/Auckland", "Pacific/Bougainville", "Pacific/Chatham", "Pacific/Chuuk", + "Pacific/Easter", "Pacific/Efate", "Pacific/Enderbury", "Pacific/Fakaofo", "Pacific/Fiji", + "Pacific/Funafuti", "Pacific/Galapagos", "Pacific/Gambier", "Pacific/Guadalcanal", "Pacific/Guam", + "Pacific/Honolulu", "Pacific/Johnston", "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", + "Pacific/Majuro", "Pacific/Marquesas", "Pacific/Midway", "Pacific/Nauru", "Pacific/Niue", "Pacific/Norfolk", + "Pacific/Noumea", "Pacific/Pago_Pago", "Pacific/Palau", "Pacific/Pitcairn", "Pacific/Pohnpei", + "Pacific/Ponape", "Pacific/Port_Moresby", "Pacific/Rarotonga", "Pacific/Saipan", "Pacific/Samoa", + "Pacific/Tahiti", "Pacific/Tarawa", "Pacific/Tongatapu", "Pacific/Truk", "Pacific/Wake", "Pacific/Wallis", + "Pacific/Yap", "Poland", "Portugal", "ROK", "Singapore", "Turkey", "UCT", "US/Alaska", "US/Aleutian", + "US/Arizona", "US/Central", "US/East-Indiana", "US/Eastern", "US/Hawaii", "US/Indiana-Starke", + "US/Michigan", "US/Mountain", "US/Pacific", "US/Samoa", "UTC", "Universal", "W-SU", "WET", "Zulu" }; + private static final String FALSE = "false"; + private static final String TRUE = "true"; + + private PrestoConstant() { + } + + public static PrestoExpression createStringConstant(String text) { + return new PrestoTextConstant(text); + } + + public static PrestoExpression createStringConstant(String text, int size) { + return new PrestoTextConstant(text, size); + } + + public static PrestoExpression createJsonConstant() { + return new PrestoJsonConstant(); + } + + public static PrestoExpression createFloatConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + assert type.getSize() == 4; + float floatValue = (float) val; + return new PrestoFloatConstant(floatValue); + } + + public static PrestoExpression createFloatConstant(double val) { + return new PrestoFloatConstant(val); + } + + public static PrestoExpression createDecimalConstant(double val) { + return new PrestoDecimalConstant(val); + } + + public static PrestoExpression createDecimalConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + int scale = type.getScale(); + int precision = type.getSize(); + BigDecimal finalBD = PrestoConstantUtils.getDecimal(val, scale, precision); + return new PrestoDecimalConstant(finalBD.doubleValue()); + } + + public static PrestoExpression createIntConstant(long val) { + return new PrestoIntConstant(val); + } + + public static PrestoExpression createIntConstant(PrestoSchema.PrestoCompositeDataType type, long val, + boolean castInteger) { + PrestoIntConstant intConstant; + assert List.of(1, 2, 4, 8).contains(type.getSize()); + switch (type.getSize()) { + case 1: + intConstant = new PrestoIntConstant((byte) val); + break; + case 2: + intConstant = new PrestoIntConstant((short) val); + break; + case 4: + intConstant = new PrestoIntConstant((int) val); + break; + default: + intConstant = new PrestoIntConstant(val); + } + if (castInteger) { + return new PrestoCastFunction(intConstant, type); + } else { + return intConstant; + } + } + + public static PrestoExpression createNullConstant() { + return new PrestoNullConstant(); + } + + public static PrestoExpression createBooleanConstant(boolean val) { + return new PrestoBooleanConstant(val); + } + + public static PrestoExpression createDateConstant(long integer) { + return new PrestoDateConstant(integer); + } + + public static PrestoExpression createTimeConstant(long integer) { + return new PrestoTimeConstant(integer); + } + + public static PrestoExpression createTimeWithTimeZoneConstant(long integer) { + return new PrestoTimeWithTimeZoneConstant(integer); + } + + public static PrestoExpression createTimestampWithTimeZoneConstant(long integer) { + return new PrestoTimestampWithTimezoneConstant(integer); + } + + public static PrestoExpression createIntervalDayToSecond(long integer) { + return new PrestoIntervalDayToSecondConstant(); + } + + public static PrestoExpression createIntervalYearToMonth(long integer) { + return new PrestoIntervalYearToMonthConstant(); + } + + public static PrestoExpression createTimestampConstant(long integer) { + return new PrestoTimestampConstant(integer); + } + + public static PrestoExpression createVarbinaryConstant(String string) { + return new PrestoVarbinaryConstant(string); + } + + public static PrestoExpression createTimezoneConstant() { + String string = Randomly.fromOptions(TIME_ZONES); + return new PrestoTextConstant(string); + } + + public static PrestoExpression createArrayConstant(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); + long size = Randomly.getNotCachedInteger(0, 10); + + List elements = new ArrayList<>(); + for (int i = 0; i <= size; i++) { + if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + elements.add(createArrayConstant(elementType)); + } else { + elements.add(generateConstant(elementType, false)); + } + } + return new PrestoArrayConstant(elements); + } + + public static PrestoExpression createMapConstant(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); + long size = Randomly.getNotCachedInteger(0, 10); + + List elements = new ArrayList<>(); + for (int i = 0; i <= size; i++) { + if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + elements.add(createArrayConstant(elementType)); + } else { + elements.add(generateConstant(elementType, false)); + } + } + return new PrestoArrayConstant(elements); + } + + public static PrestoExpression generateConstant(PrestoSchema.PrestoCompositeDataType type, boolean castInteger) { + Randomly randomly = new Randomly(); + switch (type.getPrimitiveDataType()) { + case ARRAY: + return PrestoConstant.createArrayConstant(type); + case NULL: + return PrestoConstant.createNullConstant(); + case CHAR: + return PrestoConstant.PrestoTextConstant.createStringConstant(randomly.getAlphabeticChar(), type.getSize()); + case VARCHAR: + return PrestoConstant.PrestoTextConstant.createStringConstant(randomly.getString(), type.getSize()); + case VARBINARY: + return PrestoConstant.createVarbinaryConstant(randomly.getString()); + case JSON: + return PrestoConstant.PrestoJsonConstant.createJsonConstant(); + case TIME: + return PrestoConstant.createTimeConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIME_WITH_TIME_ZONE: + return PrestoConstant.createTimeWithTimeZoneConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIMESTAMP: + return PrestoConstant.createTimestampConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIMESTAMP_WITH_TIME_ZONE: + return PrestoConstant.createTimestampWithTimeZoneConstant(randomly.getLong(0, System.currentTimeMillis())); + case INTERVAL_YEAR_TO_MONTH: + return PrestoConstant.createIntervalYearToMonth(randomly.getLong(0, System.currentTimeMillis())); + case INTERVAL_DAY_TO_SECOND: + return PrestoConstant.createIntervalDayToSecond(randomly.getLong(0, System.currentTimeMillis())); + case INT: + return PrestoConstant.PrestoIntConstant.createIntConstant(type, Randomly.getNonCachedInteger(), + castInteger); + case FLOAT: + return PrestoConstant.PrestoFloatConstant.createFloatConstant(randomly.getDouble()); + case BOOLEAN: + return PrestoConstant.PrestoBooleanConstant.createBooleanConstant(Randomly.getBoolean()); + case DATE: + return PrestoConstant.createDateConstant(randomly.getLong(0, System.currentTimeMillis())); + case DECIMAL: + return PrestoConstant.createDecimalConstant(type, randomly.getLong(0, System.currentTimeMillis())); + default: + throw new AssertionError("Unknown type: " + type); + } + } + + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isArray() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + public static class PrestoNullConstant extends PrestoConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + } + + public static class PrestoIntConstant extends PrestoConstant { + + private final long value; + + public PrestoIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + @Override + public boolean isInt() { + return true; + } + + } + + public static class PrestoFloatConstant extends PrestoConstant { + + private final double value; + + public PrestoFloatConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "infinity()"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-infinity()"; + } + return String.valueOf(value); + } + + @Override + public boolean isFloat() { + return true; + } + + @Override + public double asFloat() { + return value; + } + + } + + public static class PrestoDecimalConstant extends PrestoConstant { + + private static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("###0.0000"); + + private final double value; + + public PrestoDecimalConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return DECIMAL_FORMAT.format(value); + } + + @Override + public double asFloat() { + return value; + } + + } + + public static class PrestoTextConstant extends PrestoConstant { + + private final String value; + + public PrestoTextConstant(String value) { + this.value = value; + } + + public PrestoTextConstant(String value, int size) { + this.value = value.substring(0, Math.min(value.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class PrestoVarbinaryConstant extends PrestoConstant { + + private final String value; + + public PrestoVarbinaryConstant(String value) { + this.value = value.replace("'", ""); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return String.format("CAST ('%s' AS VARBINARY)", value); + } + + } + + public static class PrestoJsonConstant extends PrestoConstant { + + private final String value; + + public PrestoJsonConstant() { + Randomly rand = new Randomly(); + JsonValueType jvt = Randomly.fromOptions(JsonValueType.values()); + String val; + switch (jvt) { + case NULL: + val = "null"; + value = "{\"val\":" + val + "}"; + break; + case FALSE: + val = FALSE; + value = "{\"val\":" + val + "}"; + break; + case TRUE: + val = TRUE; + value = "{\"val\":" + val + "}"; + break; + case STRING: + String randString = rand.getString(); + String string = randString.substring(0, Math.min(randString.length(), 250)); + string = string.replace("'", ""); + // https://www.rfc-editor.org/rfc/rfc8259#page-8 + string = PrestoConstantUtils.removeAllControlChars(string); + string = string.replace("\\", "\\\\"); + + value = "{\"val\": \"" + string + "\"}"; + break; + case NUMBER: + if (Randomly.getBoolean()) { + int no = (int) rand.getInteger(); + val = String.valueOf(no); + } else { + double no = rand.getDouble(); + val = String.valueOf(no); + } + value = "{\"val\": " + val + "}"; + break; + case ARRAY: + value = "{\"employees\":[\"John\", \"Anna\", \"Peter\"]}"; + break; + case OBJECT: + value = "{\"employee\":{\"name\":\"John\", \"age\":30, \"city\":\"New York\"}}"; + break; + default: + value = "{}"; + } + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "JSON '" + value + "'"; + } + + private enum JsonValueType { + OBJECT, ARRAY, NUMBER, STRING, TRUE, FALSE, NULL + } + + } + + public static class PrestoDateConstant extends PrestoConstant { + + private final String textRepresentation; + + public PrestoDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepresentation); + } + + } + + public static class PrestoTimeConstant extends PrestoConstant { + + public final String textRepresentation; + + public PrestoTimeConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIME '%s'", textRepresentation); + } + + } + + public static class PrestoTimeWithTimeZoneConstant extends PrestoConstant { + + private final String textRepresentation; + private final String timeZone; + + public PrestoTimeWithTimeZoneConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + textRepresentation = dateFormat.format(timestamp); + this.timeZone = Randomly.fromOptions(TIME_ZONES); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIME '%s %s'", textRepresentation, timeZone); + } + + } + + public static class PrestoTimestampConstant extends PrestoConstant { + + private final String textRepresentation; + + public PrestoTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + this.textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepresentation); + } + + } + + public static class PrestoTimestampWithTimezoneConstant extends PrestoConstant { + + private final String textRepresentation; + private final String timeZone; + + public PrestoTimestampWithTimezoneConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + this.textRepresentation = dateFormat.format(timestamp); + this.timeZone = Randomly.fromOptions(TIME_ZONES); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s %s'", textRepresentation, timeZone); + } + + } + + public static class PrestoIntervalDayToSecondConstant extends PrestoConstant { + + private final String textRepresentation; + private final Interval fromInterval; + + public PrestoIntervalDayToSecondConstant() { + this.fromInterval = Randomly.fromOptions(Interval.values()); + SimpleDateFormat dateFormat = new SimpleDateFormat("dd HH:mm:ss"); + switch (fromInterval) { + case DAY: + dateFormat = new SimpleDateFormat("dd"); + break; + case HOUR: + dateFormat = new SimpleDateFormat("HH"); + break; + case MINUTE: + dateFormat = new SimpleDateFormat("mm"); + break; + case SECOND: + dateFormat = new SimpleDateFormat("ss"); + break; + default: + break; + } + + Randomly rand = new Randomly(); + + Timestamp timestamp = new Timestamp(rand.getLong(0, System.currentTimeMillis())); + this.textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + // if (toInterval == null) { + return String.format("INTERVAL '%s' %s", textRepresentation, fromInterval.name()); + // } else { + // return String.format("INTERVAL '%s' %s TO %s", textRepresentation, fromInterval, toInterval); + // } + } + + private enum Interval { + DAY, HOUR, MINUTE, SECOND + } + + } + + public static class PrestoIntervalYearToMonthConstant extends PrestoConstant { + + public String textRepresentation; + private final Interval fromInterval; + + public PrestoIntervalYearToMonthConstant() { + fromInterval = Randomly.fromOptions(Interval.values()); + SimpleDateFormat dateFormat; + switch (fromInterval) { + case YEAR: + dateFormat = new SimpleDateFormat("yyyy"); + break; + case MONTH: + dateFormat = new SimpleDateFormat("MM"); + break; + default: + dateFormat = new SimpleDateFormat("yyyy-MM"); + } + + Randomly rand = new Randomly(); + + Timestamp timestamp = new Timestamp(rand.getLong(0, System.currentTimeMillis())); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("INTERVAL '%s' %s", textRepresentation, fromInterval.name()); + } + + private enum Interval { + YEAR, MONTH + } + + } + + public static class PrestoBooleanConstant extends PrestoConstant { + + private final boolean value; + + public PrestoBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + } + + public static class PrestoArrayConstant extends PrestoConstant { + + private final List elements; + + public PrestoArrayConstant(List elements) { + this.elements = new ArrayList<>(elements); + } + + @Override + public boolean isArray() { + return true; + } + + @Override + public String toString() { + return "ARRAY[" + elements.stream().map(Object::toString).collect(Collectors.joining(", ")) + "]"; + } + + } + +} diff --git a/src/sqlancer/presto/ast/PrestoDateFunction.java b/src/sqlancer/presto/ast/PrestoDateFunction.java new file mode 100644 index 000000000..a8e00a3c0 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoDateFunction.java @@ -0,0 +1,523 @@ +package sqlancer.presto.ast; + +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoDateFunction implements PrestoFunction { + + // Date and Time Functions# + // Returns the current date as of the start of the query. + CURRENT_DATE("current_date", PrestoDataType.DATE), + + // Returns the current time as of the start of the query. + CURRENT_TIME("current_time", PrestoDataType.TIME_WITH_TIME_ZONE), + + // Returns the current timestamp as of the start of the query. + CURRENT_TIMESTAMP("current_timestamp", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Returns the current time zone in the format defined by IANA (e.g., America/Los_Angeles) or as fixed offset from + // UTC (e.g., +08:35) + CURRENT_TIMEZONE("current_timezone", PrestoDataType.VARCHAR), + + // This is an alias for CAST(x AS date). + DATE("date", PrestoDataType.DATE, PrestoDataType.DATE, PrestoDataType.INT, PrestoDataType.VARCHAR), + + // Returns the last day of the month. + LAST_DAY_OF_MONTH("last_day_of_month", PrestoDataType.DATE, PrestoDataType.DATE), + + // Parses the ISO 8601 formatted string into a timestamp with time zone. + FROM_ISO8601_TIMESTAMP("from_iso8601_timestamp", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.VARCHAR), + + // Parses the ISO 8601 formatted string into a date. + FROM_ISO8601_DATE("from_iso8601_date", PrestoDataType.DATE, PrestoDataType.VARCHAR), + + // Returns the UNIX timestamp unixtime as a timestamp. + FROM_UNIXTIME("from_unixtime", PrestoDataType.TIMESTAMP, PrestoDataType.INT), + + // Returns the UNIX timestamp unixtime as a timestamp with time zone using string for the time zone. + FROM_UNIXTIME_TIMEZONE("from_unixtime", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.INT, + PrestoDataType.VARCHAR) { + @Override + public boolean shouldPreserveOrderOfArguments() { + return true; + } + }, + + // Returns the UNIX timestamp unixtime as a timestamp with time zone using hours and minutes for the time zone + // offset. + FROM_UNIXTIME_HOURS_MINUTES("from_unixtime", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.INT, + PrestoDataType.INT) { + @Override + public boolean shouldPreserveOrderOfArguments() { + return true; + } + }, + + // Returns the current time as of the start of the query. -> time + LOCALTIME("localtime", PrestoDataType.TIME), + + // Returns the current timestamp as of the start of the query. -> timestamp + LOCALTIMESTAMP("localtimestamp", PrestoDataType.TIMESTAMP), + + // This is an alias for current_timestamp. → timestamp with time zone# + NOW("now", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Formats x as an ISO 8601 string. x can be date, timestamp, or timestamp with time zone. → varchar# + TO_ISO8601("to_iso8601", PrestoDataType.VARCHAR, PrestoDataType.DATE, PrestoDataType.TIMESTAMP, + PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Returns the day-to-second interval as milliseconds. → bigint# + TO_MILLISECONDS("to_milliseconds", PrestoDataType.INT, PrestoDataType.INTERVAL_DAY_TO_SECOND), + TO_MILLISECONDS_2("to_milliseconds", PrestoDataType.INT, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + + // Returns timestamp as a UNIX timestamp. → double# + TO_UNIXTIME("to_unixtime", PrestoDataType.FLOAT, PrestoDataType.TIMESTAMP), + TO_UNIXTIME_2("to_unixtime", PrestoDataType.FLOAT, PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // The following SQL-standard functions do not use parenthesis: + CURRENT_DATE_NA("current_date", PrestoDataType.DATE) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + CURRENT_TIME_NA("current_time", PrestoDataType.TIME) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + CURRENT_TIMESTAMP_NA("current_timestamp", PrestoDataType.TIMESTAMP) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + LOCALTIME_NA("localtime", PrestoDataType.TIME) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + LOCALTIMESTAMP_NA("localtimestamp", PrestoDataType.TIMESTAMP) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + // Truncation Function + // date_trunc(unit, x) → [same as input] + DATE_TRUNC_1("date_trunc", PrestoDataType.TIMESTAMP, PrestoDataType.VARCHAR, PrestoDataType.TIMESTAMP), + DATE_TRUNC_2("date_trunc", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.VARCHAR, + PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + DATE_TRUNC_3("date_trunc", PrestoDataType.DATE, PrestoDataType.VARCHAR, PrestoDataType.DATE), + DATE_TRUNC_4("date_trunc", PrestoDataType.TIME, PrestoDataType.VARCHAR, PrestoDataType.TIME); + + /* + * + * Interval Functions# The functions in this section support the following interval units: + * + * Unit + * + * Description + * + * millisecond + * + * Milliseconds + * + * second + * + * Seconds + * + * minute + * + * Minutes + * + * hour + * + * Hours + * + * day + * + * Days + * + * week + * + * Weeks + * + * month + * + * Months + * + * quarter + * + * Quarters of a year + * + * year + * + * Years + * + * date_add(unit, value, timestamp) → [same as input]# Adds an interval value of type unit to timestamp. Subtraction + * can be performed by using a negative value. + * + * date_diff(unit, timestamp1, timestamp2) → bigint# Returns timestamp2 - timestamp1 expressed in terms of unit. + * + * Duration Function# The parse_duration function supports the following units: + * + * Unit + * + * Description + * + * ns + * + * Nanoseconds + * + * us + * + * Microseconds + * + * ms + * + * Milliseconds + * + * s + * + * Seconds + * + * m + * + * Minutes + * + * h + * + * Hours + * + * d + * + * Days + * + * parse_duration(string) → interval# Parses string of format value unit into an interval, where value is fractional + * number of unit values: + * + * SELECT parse_duration('42.8ms'); -- 0 00:00:00.043 SELECT parse_duration('3.81 d'); -- 3 19:26:24.000 SELECT + * parse_duration('5m'); -- 0 00:05:00.000 MySQL Date Functions# The functions in this section use a format string + * that is compatible with the MySQL date_parse and str_to_date functions. The following table, based on the MySQL + * manual, describes the format specifiers: + * + * Specifier + * + * Description + * + * %a + * + * Abbreviated weekday name (Sun .. Sat) + * + * %b + * + * Abbreviated month name (Jan .. Dec) + * + * %c + * + * Month, numeric (1 .. 12) 4 + * + * %D + * + * Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) + * + * %d + * + * Day of the month, numeric (01 .. 31) 4 + * + * %e + * + * Day of the month, numeric (1 .. 31) 4 + * + * %f + * + * Fraction of second (6 digits for printing: 000000 .. 999000; 1 - 9 digits for parsing: 0 .. 999999999) 1 + * + * %H + * + * Hour (00 .. 23) + * + * %h + * + * Hour (01 .. 12) + * + * %I + * + * Hour (01 .. 12) + * + * %i + * + * Minutes, numeric (00 .. 59) + * + * %j + * + * Day of year (001 .. 366) + * + * %k + * + * Hour (0 .. 23) + * + * %l + * + * Hour (1 .. 12) + * + * %M + * + * Month name (January .. December) + * + * %m + * + * Month, numeric (01 .. 12) 4 + * + * %p + * + * AM or PM + * + * %r + * + * Time, 12-hour (hh:mm:ss followed by AM or PM) + * + * %S + * + * Seconds (00 .. 59) + * + * %s + * + * Seconds (00 .. 59) + * + * %T + * + * Time, 24-hour (hh:mm:ss) + * + * %U + * + * Week (00 .. 53), where Sunday is the first day of the week + * + * %u + * + * Week (00 .. 53), where Monday is the first day of the week + * + * %V + * + * Week (01 .. 53), where Sunday is the first day of the week; used with %X + * + * %v + * + * Week (01 .. 53), where Monday is the first day of the week; used with %x + * + * %W + * + * Weekday name (Sunday .. Saturday) + * + * %w + * + * Day of the week (0 .. 6), where Sunday is the first day of the week 3 + * + * %X + * + * Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V + * + * %x + * + * Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v + * + * %Y + * + * Year, numeric, four digits + * + * %y + * + * Year, numeric (two digits) 2 + * + * %% + * + * A literal % character + * + * %x + * + * x, for any x not listed above + * + * 1 Timestamp is truncated to milliseconds. + * + * 2 When parsing, two-digit year format assumes range 1970 ... 2069, so “70” will result in year 1970 but “69” will + * produce 2069. + * + * 3 This specifier is not supported yet. Consider using day_of_week() (it uses 1-7 instead of 0-6). + * + * 4(1,2,3,4) This specifier does not support 0 as a month or day. + * + * Warning + * + * The following specifiers are not currently supported: %D %U %u %V %w %X + * + * date_format(timestamp, format) → varchar# Formats timestamp as a string using format. + * + * date_parse(string, format) → timestamp# Parses string into a timestamp using format. + * + * Java Date Functions# The functions in this section use a format string that is compatible with JodaTime’s + * DateTimeFormat pattern format. + * + * format_datetime(timestamp, format) → varchar# Formats timestamp as a string using format. + * + * parse_datetime(string, format) → timestamp with time zone# Parses string into a timestamp with time zone using + * format. + * + * Extraction Function# The extract function supports the following fields: + * + * Field + * + * Description + * + * YEAR + * + * year() + * + * QUARTER + * + * quarter() + * + * MONTH + * + * month() + * + * WEEK + * + * week() + * + * DAY + * + * day() + * + * DAY_OF_MONTH + * + * day() + * + * DAY_OF_WEEK + * + * day_of_week() + * + * DOW + * + * day_of_week() + * + * DAY_OF_YEAR + * + * day_of_year() + * + * DOY + * + * day_of_year() + * + * YEAR_OF_WEEK + * + * year_of_week() + * + * YOW + * + * year_of_week() + * + * HOUR + * + * hour() + * + * MINUTE + * + * minute() + * + * SECOND + * + * second() + * + * TIMEZONE_HOUR + * + * timezone_hour() + * + * TIMEZONE_MINUTE + * + * timezone_minute() + * + * The types supported by the extract function vary depending on the field to be extracted. Most fields support all + * date and time types. + * + * extract(field FROM x) → bigint# Returns field from x. + * + * Note + * + * This SQL-standard function uses special syntax for specifying the arguments. + * + * Convenience Extraction Functions# day(x) → bigint# Returns the day of the month from x. + * + * day_of_month(x) → bigint# This is an alias for day(). + * + * day_of_week(x) → bigint# Returns the ISO day of the week from x. The value ranges from 1 (Monday) to 7 (Sunday). + * + * day_of_year(x) → bigint# Returns the day of the year from x. The value ranges from 1 to 366. + * + * dow(x) → bigint# This is an alias for day_of_week(). + * + * doy(x) → bigint# This is an alias for day_of_year(). + * + * hour(x) → bigint# Returns the hour of the day from x. The value ranges from 0 to 23. + * + * millisecond(x) → bigint# Returns the millisecond of the second from x. + * + * minute(x) → bigint# Returns the minute of the hour from x. + * + * month(x) → bigint# Returns the month of the year from x. + * + * quarter(x) → bigint# Returns the quarter of the year from x. The value ranges from 1 to 4. + * + * second(x) → bigint# Returns the second of the minute from x. + * + * timezone_hour(timestamp) → bigint# Returns the hour of the time zone offset from timestamp. + * + * timezone_minute(timestamp) → bigint# Returns the minute of the time zone offset from timestamp. + * + * week(x) → bigint# Returns the ISO week of the year from x. The value ranges from 1 to 53. + * + * week_of_year(x) → bigint# This is an alias for week(). + * + * year(x) → bigint# Returns the year from x. + * + * year_of_week(x) → bigint# Returns the year of the ISO week from x. + * + * yow(x) → bigint# This is an alias for year_of_week(). + * + * + * + */ + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoDateFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoDefaultFunction.java b/src/sqlancer/presto/ast/PrestoDefaultFunction.java new file mode 100644 index 000000000..99b7f7fde --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoDefaultFunction.java @@ -0,0 +1,232 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public enum PrestoDefaultFunction implements PrestoFunction { + + // Conditional functions + IF_TRUE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { PrestoSchema.PrestoDataType.BOOLEAN, + returnType.getPrimitiveDataType() }; + } + }, + + IF_TRUE_FALSE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { PrestoSchema.PrestoDataType.BOOLEAN, + returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + NULLIF("nullif", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + COALESCE("coalesce", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { + return super.getArgumentsForReturnType(gen, depth, argumentTypes, returnType); + } + }, + + // comparison + + // Returns the largest of the provided values. → [same as input] + GREATEST("greatest", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + }, + // Returns the smallest of the provided values. → [same as input] + LEAST("least", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private static final int UNLIMITED_NO_OF_ARGUMENTS = -1; + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoDefaultFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + } + + PrestoDefaultFunction(PrestoDataType returnType) { + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + this.functionName = toString(); + } + + PrestoDefaultFunction(PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + this.functionName = toString(); + } + + PrestoDefaultFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + public static List getFunctionsCompatibleWith(PrestoCompositeDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)).collect(Collectors.toList()); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public int getNumberOfArguments() { + return argumentTypes == null ? 0 : argumentTypes.length; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + + @Override + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { + List arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to the same element type. + PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + + if (getNumberOfArguments() == UNLIMITED_NO_OF_ARGUMENTS) { + PrestoDataType dataType = getArgumentTypes(returnType)[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoCompositeDataType type; + + if (dataType == PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(type, depth + 1)); + } + } else { + for (PrestoDataType arg : argumentTypes) { + PrestoCompositeDataType type; + if (arg == PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + type = savedArrayType; + } else { + type = PrestoCompositeDataType.fromDataType(arg); + } + arguments.add(gen.generateExpression(type, depth + 1)); + + } + } + return arguments; + } + + @Override + public String toString() { + if (functionName != null) { + return functionName; + } + return super.toString(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoExpression.java b/src/sqlancer/presto/ast/PrestoExpression.java new file mode 100644 index 000000000..52e0ed784 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoExpression.java @@ -0,0 +1,8 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.presto.PrestoSchema.PrestoColumn; + +public interface PrestoExpression extends Expression { + +} diff --git a/src/sqlancer/presto/ast/PrestoFunction.java b/src/sqlancer/presto/ast/PrestoFunction.java new file mode 100644 index 000000000..fc7ef5b79 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunction.java @@ -0,0 +1,126 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public interface PrestoFunction extends PrestoExpression { + + String getFunctionName(); + + boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType); + + PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType); + + default List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoSchema.PrestoDataType[] argumentTypes, PrestoSchema.PrestoCompositeDataType returnType) { + + List arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + // -1 - unlimited number of arguments + if (getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = argumentTypes[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType type; + + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(type, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType arg : argumentTypes) { + PrestoSchema.PrestoCompositeDataType dataType; + if (arg == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + dataType = savedArrayType; + } else { + dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); + } + PrestoExpression expression = gen.generateExpression(dataType, depth + 1); + arguments.add(expression); + } + } + return arguments; + } + + default List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoSchema.PrestoCompositeDataType returnType, boolean orderable) { + + List arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + if (getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = getArgumentTypes(returnType)[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType compositeDataType; + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + compositeDataType = savedArrayType; + } else { + compositeDataType = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(compositeDataType, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType dataType : getArgumentTypes(returnType)) { + PrestoSchema.PrestoCompositeDataType compositeDataType; + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + PrestoSchema.PrestoCompositeDataType arrayType; + do { + arrayType = dataType.get(); + } while (!arrayType.getElementType().isOrderable()); + savedArrayType = arrayType; + } + compositeDataType = savedArrayType; + } else { + compositeDataType = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + PrestoExpression expression = gen.generateExpression(compositeDataType, depth + 1); + arguments.add(expression); + } + } + return arguments; + } + + default int getNumberOfArguments() { + return getArgumentTypes(null).length; + } + + default boolean shouldPreserveOrderOfArguments() { + return false; + } + + default boolean isStandardFunction() { + return true; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoFunctionNode.java b/src/sqlancer/presto/ast/PrestoFunctionNode.java new file mode 100644 index 000000000..14409a825 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunctionNode.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class PrestoFunctionNode extends NewFunctionNode implements PrestoExpression { + public PrestoFunctionNode(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java new file mode 100644 index 000000000..88007053d --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java @@ -0,0 +1,23 @@ +package sqlancer.presto.ast; + +import sqlancer.presto.PrestoSchema; + +public class PrestoFunctionWithoutParenthesis implements PrestoExpression { + + private final PrestoSchema.PrestoCompositeDataType type; + private final String expr; + + public PrestoFunctionWithoutParenthesis(String expr, PrestoSchema.PrestoCompositeDataType type) { + this.expr = expr; + this.type = type; + } + + public String getExpr() { + return expr; + } + + public PrestoSchema.PrestoCompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoInOperation.java b/src/sqlancer/presto/ast/PrestoInOperation.java new file mode 100644 index 000000000..95edd7fc3 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoInOperation.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class PrestoInOperation extends NewInOperatorNode implements PrestoExpression { + public PrestoInOperation(PrestoExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/presto/ast/PrestoJoin.java b/src/sqlancer/presto/ast/PrestoJoin.java new file mode 100644 index 000000000..3fe82fc37 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoJoin.java @@ -0,0 +1,120 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoJoin implements PrestoExpression, Join { + + private final PrestoTableReference leftTable; + private final PrestoTableReference rightTable; + private final JoinType joinType; + private PrestoExpression onCondition; + private OuterType outerType; + + public PrestoJoin(PrestoTableReference leftTable, PrestoTableReference rightTable, JoinType joinType, + PrestoExpression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public static List getJoins(List tableList, PrestoGlobalState globalState) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + PrestoTableReference leftTable = tableList.remove(0); + PrestoTableReference rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + PrestoTypedExpressionGenerator joinGen = new PrestoTypedExpressionGenerator(globalState) + .setColumns(columns); + switch (JoinType.getRandom()) { + case INNER: + joinExpressions.add(PrestoJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + case LEFT: + joinExpressions.add(PrestoJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + case RIGHT: + joinExpressions.add(PrestoJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static PrestoJoin createRightOuterJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { + return new PrestoJoin(left, right, JoinType.RIGHT, predicate); + } + + public static PrestoJoin createLeftOuterJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { + return new PrestoJoin(left, right, JoinType.LEFT, predicate); + } + + public static PrestoJoin createInnerJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { + return new PrestoJoin(left, right, JoinType.INNER, predicate); + } + + public PrestoTableReference getLeftTable() { + return leftTable; + } + + public PrestoTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public PrestoExpression getOnCondition() { + return onCondition; + } + + public OuterType getOuterType() { + return outerType; + } + + @SuppressWarnings("unused") + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public enum JoinType { + INNER, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + FULL, LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + + @Override + public void setOnClause(PrestoExpression onClause) { + onCondition = onClause; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java new file mode 100644 index 000000000..20ceaee79 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java @@ -0,0 +1,37 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +public class PrestoMultiValuedComparison implements PrestoExpression { + + private final PrestoExpression left; + private final List right; + private final PrestoMultiValuedComparisonType type; + private final PrestoMultiValuedComparisonOperator op; + + public PrestoMultiValuedComparison(PrestoExpression left, List right, + PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { + this.left = left; + this.right = new ArrayList<>(right); + this.type = type; + this.op = op; + } + + public PrestoExpression getLeft() { + return left; + } + + public PrestoMultiValuedComparisonOperator getOp() { + return op; + } + + public List getRight() { + return new ArrayList<>(right); + } + + public PrestoMultiValuedComparisonType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java new file mode 100644 index 000000000..829f554f4 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java @@ -0,0 +1,43 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; + +public enum PrestoMultiValuedComparisonOperator { + EQUALS("="), NOT_EQUALS("<>"), NOT_EQUALS_ALT("!="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), + SMALLER_EQUALS("<="); + + private final String stringRepresentation; + + PrestoMultiValuedComparisonOperator(String stringRepresentation) { + this.stringRepresentation = stringRepresentation; + } + + public static PrestoMultiValuedComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public static PrestoMultiValuedComparisonOperator getRandomForType(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoDataType dataType = type.getPrimitiveDataType(); + + switch (dataType) { + case BOOLEAN: + case INT: + case FLOAT: + case DECIMAL: + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return getRandom(); + default: + return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT); + } + } + + public String getStringRepresentation() { + return stringRepresentation; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java new file mode 100644 index 000000000..d1ecdd382 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; + +public enum PrestoMultiValuedComparisonType { + ANY, SOME, ALL; + + public static PrestoMultiValuedComparisonType getRandom() { + return Randomly.fromOptions(values()); + } +} diff --git a/src/sqlancer/presto/ast/PrestoPostfixText.java b/src/sqlancer/presto/ast/PrestoPostfixText.java new file mode 100644 index 000000000..de99fede4 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoPostfixText.java @@ -0,0 +1,9 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class PrestoPostfixText extends NewPostfixTextNode implements PrestoExpression { + public PrestoPostfixText(PrestoExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java new file mode 100644 index 000000000..3cbccc7b2 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java @@ -0,0 +1,34 @@ +package sqlancer.presto.ast; + +public class PrestoQuantifiedComparison implements PrestoExpression { + + private final PrestoExpression left; + private final PrestoSelect right; + private final PrestoMultiValuedComparisonType type; + private final PrestoMultiValuedComparisonOperator op; + + public PrestoQuantifiedComparison(PrestoExpression left, PrestoSelect right, PrestoMultiValuedComparisonType type, + PrestoMultiValuedComparisonOperator op) { + this.left = left; + this.right = right; + this.type = type; + this.op = op; + } + + public PrestoExpression getLeft() { + return left; + } + + public PrestoMultiValuedComparisonOperator getOp() { + return op; + } + + public PrestoExpression getRight() { + return right; + } + + public PrestoMultiValuedComparisonType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoSelect.java b/src/sqlancer/presto/ast/PrestoSelect.java new file mode 100644 index 000000000..f1eb50186 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoSelect.java @@ -0,0 +1,41 @@ +package sqlancer.presto.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; + +public class PrestoSelect extends SelectBase + implements PrestoExpression, Select { + + private boolean isDistinct; + + public boolean isDistinct() { + return isDistinct; + } + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (PrestoExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (PrestoJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return PrestoToStringVisitor.asString(this); + } +} diff --git a/src/sqlancer/presto/ast/PrestoTableReference.java b/src/sqlancer/presto/ast/PrestoTableReference.java new file mode 100644 index 000000000..7552ecc64 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoTableReference.java @@ -0,0 +1,12 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.presto.PrestoSchema; + +public class PrestoTableReference extends TableReferenceNode + implements PrestoExpression { + + public PrestoTableReference(PrestoSchema.PrestoTable table) { + super(table); + } +} diff --git a/src/sqlancer/presto/ast/PrestoTernary.java b/src/sqlancer/presto/ast/PrestoTernary.java new file mode 100644 index 000000000..618daa0fc --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoTernary.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewTernaryNode; + +public class PrestoTernary extends NewTernaryNode implements PrestoExpression { + public PrestoTernary(PrestoExpression left, PrestoExpression middle, PrestoExpression right, String leftStr, + String rightStr) { + super(left, middle, right, leftStr, rightStr); + } +} diff --git a/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java new file mode 100644 index 000000000..99677d3f8 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java @@ -0,0 +1,52 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.presto.PrestoSchema; + +public class PrestoUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements PrestoExpression { + + public PrestoUnaryPostfixOperation(PrestoExpression expr, PrestoUnaryPostfixOperator op) { + super(expr, op); + } + + public PrestoExpression getExpression() { + return getExpr(); + } + + public enum PrestoUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public PrestoSchema.PrestoDataType[] getInputDataTypes() { + return PrestoSchema.PrestoDataType.values(); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public PrestoSchema.PrestoDataType[] getInputDataTypes() { + return PrestoSchema.PrestoDataType.values(); + } + }; + + private final String textRepresentations; + + PrestoUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static PrestoUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract PrestoSchema.PrestoDataType[] getInputDataTypes(); + + } + +} diff --git a/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java new file mode 100644 index 000000000..db7a08068 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java @@ -0,0 +1,62 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.presto.PrestoSchema; + +public class PrestoUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements PrestoExpression { + + public PrestoUnaryPrefixOperation(PrestoExpression expression, BinaryOperatorNode.Operator operation) { + super(expression, operation); + } + + public enum PrestoUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", PrestoSchema.PrestoDataType.BOOLEAN) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.BOOLEAN; + } + }, + + UNARY_PLUS("+", PrestoSchema.PrestoDataType.INT, PrestoSchema.PrestoDataType.FLOAT, + PrestoSchema.PrestoDataType.DECIMAL) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.INT; + } + }, + UNARY_MINUS("-", PrestoSchema.PrestoDataType.INT, PrestoSchema.PrestoDataType.FLOAT, + PrestoSchema.PrestoDataType.DECIMAL) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.INT; + } + }; + + private final String textRepresentation; + private final PrestoSchema.PrestoDataType[] dataTypes; + + PrestoUnaryPrefixOperator(String textRepresentation, PrestoSchema.PrestoDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public PrestoSchema.PrestoDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + public abstract PrestoSchema.PrestoDataType getExpressionType(); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + + public PrestoSchema.PrestoDataType getExpressionType(PrestoSchema.PrestoDataType type) { + return type; + } + } + +} diff --git a/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java b/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java new file mode 100644 index 000000000..f3d5f9d55 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java @@ -0,0 +1,67 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoTable; + +public final class PrestoAlterTableGenerator { + + private PrestoAlterTableGenerator() { + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + // PrestoTypedExpressionGenerator gen = new + // PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(PrestoCompositeDataType.getRandomWithoutNull()); + break; + case ALTER_COLUMN: + sb.append("ALTER COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" SET DATA TYPE "); + sb.append(PrestoCompositeDataType.getRandomWithoutNull()); + // if (Randomly.getBoolean()) { + // sb.append(" USING "); + // PrestoErrors.addExpressionErrors(errors); + // sb.append(PrestoToStringVisitor.asString(gen.generateExpression())); + // } + errors.add("Cannot change the type of this column: an index depends on it!"); + errors.add("Cannot change the type of a column that has a UNIQUE or PRIMARY KEY constraint specified"); + errors.add("Unimplemented type for cast"); + errors.add("Conversion:"); + errors.add("Cannot change the type of a column that has a CHECK constraint specified"); + break; + case DROP_COLUMN: + sb.append("DROP COLUMN "); + sb.append(table.getRandomColumn().getName()); + errors.add("named in key does not exist"); // TODO + errors.add("Cannot drop this column:"); + errors.add("Cannot drop column: table only has one column remaining!"); + errors.add("because there is a CHECK constraint that depends on it"); + errors.add("because there is a UNIQUE constraint that depends on it"); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN + } + +} diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java new file mode 100644 index 000000000..4d5724992 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -0,0 +1,37 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; + +public final class PrestoDeleteGenerator extends AbstractDeleteGenerator { + + private final PrestoGlobalState globalState; + + private PrestoDeleteGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canonicalizeString = false; + } + + public static SQLQueryAdapter generate(PrestoGlobalState globalState) { + return new PrestoDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + appendDeleteFromTable(table.getName()); + if (Randomly.getBoolean()) { + appendWhereClause(PrestoToStringVisitor + .asString(new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); + } + PrestoErrors.addExpressionErrors(errors); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java new file mode 100644 index 000000000..2d7b75cf8 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -0,0 +1,61 @@ +package sqlancer.presto.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoIndexGenerator extends AbstractIndexGenerator { + + private final PrestoGlobalState globalState; + + public PrestoIndexGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + this.canonicalizeString = false; + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + return new PrestoIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { + errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + } + appendCreateIndex(unique); + sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + sb.append(" ON "); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append("("); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + if (Randomly.getBoolean()) { + PrestoExpression expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + appendWhereClause(PrestoToStringVisitor.asString(expr)); + } + errors.add("already exists!"); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java new file mode 100644 index 000000000..15d5f3543 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -0,0 +1,43 @@ +package sqlancer.presto.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoInsertGenerator extends AbstractInsertGenerator { + + private final PrestoGlobalState globalState; + + public PrestoInsertGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canonicalizeString = false; + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + return new PrestoInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + buildInsertInto(table.getName(), columns); + PrestoErrors.addInsertErrors(errors); + } + + @Override + protected void insertValue(PrestoColumn prestoColumn) { + PrestoExpression constant = new PrestoTypedExpressionGenerator(globalState) + .generateInsertConstant(prestoColumn.getType()); + sb.append(PrestoToStringVisitor.asString(constant)); + + } + +} diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java new file mode 100644 index 000000000..5c3c0db82 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -0,0 +1,73 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; + +public final class PrestoRandomQuerySynthesizer { + + private PrestoRandomQuerySynthesizer() { + } + + public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrColumns) { + PrestoTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); + PrestoSelect select = new PrestoSelect(); + // TODO: distinct + // select.setDistinct(Randomly.getBoolean()); + // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); + List columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + // if (allowAggregates && Randomly.getBoolean()) { + PrestoExpression expression = gen + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + columns.add(expression); + // } else { + // columns.add(gen()); + // } + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, globalState).stream() + .collect(Collectors.toList()); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableList)); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull())); + } + if (Randomly.getBoolean()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + if (Randomly.getBoolean()) { + select.setLimitClause(PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + // if (Randomly.getBoolean()) { + // select.setOffsetClause( + // PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, + // Integer.MAX_VALUE))); + // } + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; + } + +} diff --git a/src/sqlancer/presto/gen/PrestoTableGenerator.java b/src/sqlancer/presto/gen/PrestoTableGenerator.java new file mode 100644 index 000000000..49e1346ad --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoTableGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; + +public class PrestoTableGenerator extends AbstractTableGenerator { + + private PrestoGlobalState globalState; + + public PrestoTableGenerator() { + this.canAffectSchema = true; + this.canonicalizeString = false; + } + + public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { + String catalog = globalState.getDbmsSpecificOptions().catalog; + String schema = globalState.getDatabaseName(); + String tableName = globalState.getSchema().getFreeTableName(); + String qualifiedName = catalog + "." + schema + "." + tableName; + appendCreateTable(qualifiedName); + appendColumnDefinitions(getNewColumns()); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + PrestoCompositeDataType columnType = PrestoCompositeDataType.getRandomWithoutNull(); + columns.add(new PrestoColumn(columnName, columnType, false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java new file mode 100644 index 000000000..4a788f01b --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -0,0 +1,887 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoAggregateFunction; +import sqlancer.presto.ast.PrestoAtTimeZoneOperator; +import sqlancer.presto.ast.PrestoBetweenOperation; +import sqlancer.presto.ast.PrestoBinaryOperation; +import sqlancer.presto.ast.PrestoCaseOperation; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoColumnReference; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoDefaultFunction; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionNode; +import sqlancer.presto.ast.PrestoInOperation; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoMultiValuedComparison; +import sqlancer.presto.ast.PrestoMultiValuedComparisonOperator; +import sqlancer.presto.ast.PrestoMultiValuedComparisonType; +import sqlancer.presto.ast.PrestoPostfixText; +import sqlancer.presto.ast.PrestoQuantifiedComparison; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; +import sqlancer.presto.ast.PrestoTernary; +import sqlancer.presto.ast.PrestoUnaryPostfixOperation; +import sqlancer.presto.ast.PrestoUnaryPrefixOperation; + +public final class PrestoTypedExpressionGenerator extends + TypedExpressionGenerator + implements NoRECGenerator, + TLPWhereGenerator { + + private final Randomly randomly; + private final PrestoGlobalState globalState; + private final int maxDepth; + private List tables; + + public PrestoTypedExpressionGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.randomly = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + @Override + public PrestoExpression generatePredicate() { + return generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), + randomly.getInteger(0, maxDepth)); + } + + @Override + public PrestoExpression negatePredicate(PrestoExpression predicate) { + return new PrestoUnaryPrefixOperation(predicate, PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT); + } + + @Override + public PrestoExpression isNull(PrestoExpression expr) { + return new PrestoUnaryPostfixOperation(expr, PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); + } + + @Override + public PrestoExpression generateConstant(PrestoSchema.PrestoCompositeDataType type) { + if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { + return PrestoConstant.createArrayConstant(type); + // case MAP: + // return PrestoConstant.createMapConstant(type); + } + return PrestoConstant.generateConstant(type, false); + } + + public PrestoExpression generateInsertConstant(PrestoSchema.PrestoCompositeDataType type) { + if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { + return PrestoConstant.createArrayConstant(type); + // case MAP: + // return PrestoConstant.createMapConstant(type); + } + return PrestoConstant.generateConstant(type, true); + } + + @Override + public PrestoExpression generateExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (allowAggregates && Randomly.getBoolean()) { + return generateAggregate(type); + } + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } else { + // TODO: functions + List applicableFunctions = PrestoDefaultFunction.getFunctionsCompatibleWith(type); + if (Randomly.getBooleanWithRatherLowProbability() && !applicableFunctions.isEmpty()) { + PrestoDefaultFunction function = Randomly.fromList(applicableFunctions); + return generateFunction(type, depth, function); + } + // TODO: try + // if (Randomly.getBooleanWithRatherLowProbability()) { + // return generateTry(type, depth); + // } + + // TODO: cast + // + // if (Randomly.getBooleanWithRatherLowProbability()) { + // Node expressionNode = generateCast(type, depth); + // } + if (Randomly.getBooleanWithRatherLowProbability()) { + return getCase(type, depth); + } + switch (type.getPrimitiveDataType()) { + case BOOLEAN: + return generateBooleanExpression(depth); + case VARCHAR: + case CHAR: + return generateStringExpression(type, depth); + case INT: + case DECIMAL: + case FLOAT: + return generateNumericExpression(depth); + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return generateTemporalExpression(type, depth); + case INTERVAL_YEAR_TO_MONTH: + case INTERVAL_DAY_TO_SECOND: + return generateIntervalExpression(type, depth); + case JSON: + return generateJsonExpression(type); + case VARBINARY: + case ARRAY: + // case MAP: + return generateLeafNode(type); // TODO + default: + throw new AssertionError(type); + } + } + } + + private PrestoExpression generateJsonExpression(PrestoSchema.PrestoCompositeDataType type) { + return generateLeafNode(type); + } + + private PrestoExpression generateCast(PrestoSchema.PrestoCompositeDataType type, int depth) { + // check can cast + PrestoExpression expressionNode = generateExpression(getRandomType(), depth + 1); + return new PrestoCastFunction(expressionNode, type); + } + + @SuppressWarnings("unused") + private PrestoExpression generateTry(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (type.getPrimitiveDataType().isNumeric() && Randomly.getBooleanWithRatherLowProbability()) { + PrestoExpression expression = generateExpression(type); + return new PrestoFunctionNode<>(List.of(expression), "try"); + } + + List applicableFunctions = PrestoDefaultFunction.getFunctionsCompatibleWith(type); + if (Randomly.getBooleanWithRatherLowProbability() && !applicableFunctions.isEmpty()) { + PrestoDefaultFunction function = Randomly.fromList(applicableFunctions); + PrestoExpression expression = generateFunction(type, depth, function); + return new PrestoFunctionNode<>(List.of(expression), "try"); + } + return new PrestoFunctionNode<>(List.of(generateCast(type, depth)), "try"); + } + + private PrestoCaseOperation getCase(PrestoSchema.PrestoCompositeDataType type, int depth) { + List conditions = new ArrayList<>(); + List cases = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + conditions.add(generateExpression(type, depth + 1)); + cases.add(generateExpression(type, depth + 1)); + } + PrestoExpression elseExpr = null; + if (Randomly.getBoolean()) { + elseExpr = generateExpression(type, depth + 1); + } + PrestoExpression expression = generateExpression(type); + return new PrestoCaseOperation(expression, conditions, cases, elseExpr); + } + + private PrestoExpression generateFunction(PrestoSchema.PrestoCompositeDataType returnType, int depth, + PrestoDefaultFunction function) { + + PrestoSchema.PrestoDataType[] argumentTypes = function.getArgumentTypes(returnType); + List arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + if (function.getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = argumentTypes[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType type; + + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(generateExpression(type, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType arg : argumentTypes) { + PrestoSchema.PrestoCompositeDataType dataType; + if (arg == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + dataType = savedArrayType; + } else { + dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); + } + PrestoExpression expression = generateExpression(dataType, depth + 1); + arguments.add(expression); + } + } + return new PrestoFunctionNode<>(arguments, function); + } + + private PrestoExpression generateStringExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } + return getStringOperation(depth); + } + + private PrestoBinaryOperation getStringOperation(int depth) { + StringExpression exprType = Randomly.fromOptions(StringExpression.values()); + if (Objects.requireNonNull(exprType) == StringExpression.CONCAT) { + PrestoExpression left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + PrestoExpression right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + PrestBinaryStringOperator operator = PrestBinaryStringOperator.CONCAT; + return new PrestoBinaryOperation(left, right, operator); + } + throw new AssertionError(exprType); + } + + private PrestoExpression generateBooleanExpression(int depth) { + List booleanExpressions = Arrays.stream(BooleanExpression.values()) + .collect(Collectors.toList()); + if (!globalState.getDbmsSpecificOptions().testBetween) { + booleanExpressions.remove(BooleanExpression.BETWEEN); + } + + booleanExpressions.remove(BooleanExpression.REGEX); + + BooleanExpression exprType = Randomly.fromList(booleanExpressions); + switch (exprType) { + case NOT: + return generateNOT(depth + 1); + case BINARY_COMPARISON: + return getBinaryComparison(depth); + case BINARY_LOGICAL: + return getBinaryLogical(depth); + case AND_OR_CHAIN: + return getAndOrChain(depth); + case REGEX: + return getRegex(depth); + case IS_NULL: + return new PrestoUnaryPostfixOperation(generateExpression(getRandomType(), depth + 1), + Randomly.fromOptions(PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL, + PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NOT_NULL)); + case IN: + return getInOperation(depth); + case BETWEEN: + return getBetween(depth); + case LIKE: + return getLike(depth); + case MULTI_VALUED_COMPARISON: // TODO other operators + return getMultiValuedComparison(depth); + default: + throw new AssertionError(exprType); + } + } + + private PrestoExpression getMultiValuedComparison(int depth) { + + PrestoSchema.PrestoCompositeDataType type; + do { + type = PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getOrderableTypes())); + } while (type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY + && !type.getElementType().getPrimitiveDataType().isOrderable()); + + PrestoMultiValuedComparisonType comparisonType = PrestoMultiValuedComparisonType.getRandom(); + PrestoMultiValuedComparisonOperator comparisonOperator = PrestoMultiValuedComparisonOperator + .getRandomForType(type); + PrestoExpression left = generateExpression(type, depth + 1); + // sub-query + PrestoSchema.PrestoCompositeDataType finalType = type; + List columnsOfType = columns.stream().filter(c -> c.getType() == finalType) + .collect(Collectors.toList()); + if (Randomly.getBooleanWithRatherLowProbability() && !columnsOfType.isEmpty()) { + PrestoSchema.PrestoColumn column = Randomly.fromList(columnsOfType); + PrestoSelect subquery = generateSubquery(List.of(column)); + return new PrestoQuantifiedComparison(left, subquery, comparisonType, comparisonOperator); + } + int nr = Randomly.smallNumber() + 2; + List rightList = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + rightList.add(generateConstant(type)); + } + return new PrestoMultiValuedComparison(left, rightList, comparisonType, comparisonOperator); + } + + private PrestoSelect generateSubquery(List columns) { + PrestoSelect select = new PrestoSelect(); + List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + List tables = columns.stream().map(AbstractTableColumn::getTable) + .collect(Collectors.toList()); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)).distinct() + .collect(Collectors.toList()); + List tableNodeList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableNodeList); + TypedExpressionGenerator typedExpressionGenerator = new PrestoTypedExpressionGenerator( + globalState).setColumns(columns); + PrestoExpression predicate = typedExpressionGenerator.generatePredicate(); + select.setWhereClause(predicate); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(typedExpressionGenerator.generateOrderBys()); + } + List joins = PrestoJoin.getJoins(tableList, globalState).stream() + .collect(Collectors.toList()); + select.setJoinList(joins); + return select; + } + + private PrestoExpression generateNumericExpression(int depth) { + PrestoSchema.PrestoDataType dataType = Randomly.fromList(PrestoSchema.PrestoDataType.getNumberTypes()); + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + if (Randomly.getBoolean()) { + BinaryOperatorNode.Operator operator = PrestoBinaryArithmeticOperator.getRandom(); + PrestoExpression left = generateExpression(type, depth); + PrestoExpression right = generateExpression(type, depth); + return new PrestoBinaryOperation(left, right, operator); + } else { + BinaryOperatorNode.Operator operator = PrestoUnaryArithmeticOperator.MINUS; + PrestoExpression left = generateExpression(type, depth); + return new PrestoUnaryPrefixOperation(left, operator); + } + } + + private PrestoExpression generateTemporalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (Randomly.getBooleanWithSmallProbability()) { + PrestoExpression left = generateExpression(type, depth); + PrestoExpression right = generateExpression(PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getIntervalTypes())), depth); + BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); + return new PrestoBinaryOperation(left, right, operator); + } + + // timestamp at time zone + if (Randomly.getBooleanWithSmallProbability() + && (type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.TIMESTAMP + || type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.TIMESTAMP_WITH_TIME_ZONE)) { + return new PrestoAtTimeZoneOperator(generateExpression(type, depth + 1), + PrestoConstant.createTimezoneConstant()); + } + return generateLeafNode(type); + } + + private PrestoExpression generateIntervalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (Randomly.getBooleanWithSmallProbability()) { + PrestoExpression left = generateExpression(type, depth); + + PrestoExpression right; + if (Randomly.getBoolean()) { + right = generateExpression(PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getTemporalTypes())), depth); + } else { + right = generateExpression(type, depth); + } + BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); + if (Randomly.getBoolean()) { + return new PrestoBinaryOperation(left, right, operator); + } else { + return new PrestoBinaryOperation(right, left, operator); + } + } + return generateLeafNode(type); + + // functions + + // timestamp at time zone + } + + private PrestoExpression getLike(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.VARCHAR); + PrestoExpression expression = generateExpression(type, depth + 1); + PrestoExpression pattern = generateExpression(type, depth + 1); + if (Randomly.getBoolean()) { + return new PrestoBinaryOperation(expression, pattern, PrestoLikeOperator.getRandom()); + } else { + String randomlyString = randomly.getString(); + String randomlyChar = randomly.getChar(); + PrestoExpression escape = new PrestoConstant.PrestoTextConstant(randomlyChar, 1); + int index = randomlyString.indexOf(randomlyChar); + while (index > -1) { + String wildcard = Randomly.fromOptions("%", "_"); + randomlyString = randomlyString.substring(0, index + 1) + wildcard + + randomlyString.substring(index + 1); + index = randomlyString.indexOf(randomlyChar, index + 1); + } + PrestoConstant.PrestoTextConstant patternString = new PrestoConstant.PrestoTextConstant(randomlyString); + return new PrestoTernary(expression, patternString, escape, "LIKE", "ESCAPE"); + } + } + + private PrestoBinaryOperation getRegex(int depth) { + PrestoExpression left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + PrestoExpression right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + return new PrestoBinaryOperation(left, right, PrestoBinaryLogicalOperator.getRandom()); + } + + private PrestoBinaryOperation getBinaryLogical(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.BOOLEAN); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); + BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); + return new PrestoBinaryOperation(left, right, operator); + } + + private PrestoExpression getBetween(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getNumericTypes())); + PrestoExpression expression = generateExpression(type, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); + return new PrestoBetweenOperation(expression, left, right, Randomly.getBoolean()); + } + + private PrestoExpression getInOperation(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.getRandomWithoutNull()); + PrestoExpression left = generateExpression(type, depth + 1); + List inList = generateExpressions(type, Randomly.smallNumber() + 1, depth + 1); + boolean isNegated = Randomly.getBoolean(); + return new PrestoInOperation(left, inList, isNegated); + } + + private PrestoExpression getAndOrChain(int depth) { + PrestoExpression left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + PrestoExpression right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); + BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); + left = new PrestoBinaryOperation(left, right, operator); + } + return left; + } + + private PrestoExpression getBinaryComparison(int depth) { + PrestoSchema.PrestoCompositeDataType type = getRandomType(); + BinaryOperatorNode.Operator op = PrestoBinaryComparisonOperator.getRandomForType(type); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); + return new PrestoBinaryOperation(left, right, op); + } + + private PrestoExpression generateNOT(int depth) { + PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator operator = PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT; + return new PrestoUnaryPrefixOperation( + generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth), + operator); + } + + @Override + protected PrestoExpression generateColumn(PrestoSchema.PrestoCompositeDataType type) { + List columnList = columns.stream() + .filter(c -> c.getType().getPrimitiveDataType() == type.getPrimitiveDataType()) + .collect(Collectors.toList()); + PrestoSchema.PrestoColumn column = Randomly.fromList(columnList); + return new PrestoColumnReference(column); + } + + @Override + public PrestoExpression generateLeafNode(PrestoSchema.PrestoCompositeDataType type) { + if (Randomly.getBoolean()) { + return generateConstant(type); + } else { + List columnList = filterColumns(type.getPrimitiveDataType()); + if (columnList.isEmpty()) { + return generateConstant(type); + } else { + return generateColumn(type); + } + } + } + + private List filterColumns(PrestoSchema.PrestoDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + @Override + protected PrestoSchema.PrestoCompositeDataType getRandomType() { + return PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull(); + } + + @Override + protected boolean canGenerateColumnOfType(PrestoSchema.PrestoCompositeDataType type) { + return columns.stream().anyMatch(c -> c.getType() == type); + } + + public PrestoExpression generateAggregate() { + PrestoAggregateFunction aggregateFunction = PrestoAggregateFunction.getRandom(); + List argsForAggregate = generateArgsForAggregate(aggregateFunction); + return new PrestoFunctionNode<>(argsForAggregate, aggregateFunction); + } + + public List generateArgsForAggregate(PrestoAggregateFunction aggregateFunction) { + PrestoSchema.PrestoCompositeDataType returnType; + do { + returnType = aggregateFunction.getCompositeReturnType(); + } while (!aggregateFunction.isCompatibleWithReturnType(returnType)); + return aggregateFunction.getArgumentsForReturnType(this, this.maxDepth - 1, returnType, false); + } + + private PrestoExpression generateAggregate(PrestoSchema.PrestoCompositeDataType type) { + PrestoAggregateFunction aggregateFunction = Randomly + .fromList(PrestoAggregateFunction.getFunctionsCompatibleWith(type)); + List argsForAggregate = generateArgsForAggregate(type, aggregateFunction); + return new PrestoFunctionNode<>(argsForAggregate, aggregateFunction); + } + + public List generateArgsForAggregate(PrestoSchema.PrestoCompositeDataType type, + PrestoAggregateFunction aggregateFunction) { + List returnTypes = aggregateFunction.getReturnTypes(type.getPrimitiveDataType()); + List arguments = new ArrayList<>(); + allowAggregates = false; // + for (PrestoSchema.PrestoDataType argumentType : returnTypes) { + arguments.add(generateExpression(PrestoSchema.PrestoCompositeDataType.fromDataType(argumentType))); + } + // return new NewFunctionNode<>(arguments, aggregateFunction); + return arguments; + } + + @Override + public List generateOrderBys() { + List expressions = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + ArrayList prestoColumns = new ArrayList<>(columns); + prestoColumns.removeIf(c -> !c.isOrderable()); + for (int i = 0; i < nr && !prestoColumns.isEmpty(); i++) { + PrestoSchema.PrestoColumn randomColumn = Randomly.fromList(prestoColumns); + PrestoColumnReference columnReference = new PrestoColumnReference(randomColumn); + prestoColumns.remove(randomColumn); + expressions.add(columnReference); + } + return expressions; + } + + public PrestoExpression generateHavingClause() { + allowAggregates = true; + PrestoExpression expr = generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + allowAggregates = false; + return expr; + } + + public PrestoExpression generateExpressionWithColumns(List columns, int remainingDepth) { + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + } + PrestoSchema.PrestoColumn column = Randomly.fromList(columns); + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + return new PrestoColumnReference(column); + } + List possibleOptions = new ArrayList<>( + Arrays.asList(PrestoTypedExpressionGenerator.Expression.values())); + PrestoTypedExpressionGenerator.Expression expr = Randomly.fromList(possibleOptions); + BinaryOperatorNode.Operator op; + switch (expr) { + case BINARY_LOGICAL: + case BINARY_ARITHMETIC: + op = PrestoTypedExpressionGenerator.PrestoBinaryLogicalOperator.getRandom(); + break; + case BINARY_COMPARISON: + op = PrestoBinaryComparisonOperator.getRandom(); + break; + default: + throw new AssertionError(); + } + return new PrestoBinaryOperation(generateExpression(column.getType(), remainingDepth - 1), + generateExpression(column.getType(), remainingDepth - 1), op); + } + + private enum StringExpression { + CONCAT + } + + public enum PrestBinaryStringOperator implements BinaryOperatorNode.Operator { + CONCAT("||"); + + private final String textRepresentation; + + PrestBinaryStringOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryTemporalOperator implements BinaryOperatorNode.Operator { + ADD("+"), SUB("-"); + + private final String textRepresentation; + + PrestoBinaryTemporalOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + private enum BooleanExpression { + NOT, BINARY_COMPARISON, BINARY_LOGICAL, AND_OR_CHAIN, REGEX, IS_NULL, IN, BETWEEN, LIKE, MULTI_VALUED_COMPARISON + } + + public enum PrestoBinaryLogicalOperator implements BinaryOperatorNode.Operator { + + AND, OR; + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + + public enum PrestoLikeOperator implements BinaryOperatorNode.Operator { + LIKE("LIKE"), // + NOT_LIKE("NOT LIKE"); + + private final String textRepresentation; + + PrestoLikeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static PrestoLikeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("="), NOT_EQUALS("<>"), NOT_EQUALS_ALT("!="), IS_DISTINCT_FROM("IS DISTINCT FROM"), + IS_NOT_DISTINCT_FROM("IS NOT DISTINCT FROM"), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), + SMALLER_EQUALS("<="); + + private final String textRepresentation; + + PrestoBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + public static BinaryOperatorNode.Operator getRandomStringOperator() { + return Randomly.fromOptions(EQUALS, NOT_EQUALS, IS_DISTINCT_FROM, IS_NOT_DISTINCT_FROM); + } + + public static BinaryOperatorNode.Operator getRandomForType(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoDataType dataType = type.getPrimitiveDataType(); + + switch (dataType) { + case BOOLEAN: + case INT: + case FLOAT: + case DECIMAL: + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return getRandom(); + case VARCHAR: + case CHAR: + case VARBINARY: + case JSON: + case ARRAY: + case INTERVAL_YEAR_TO_MONTH: + case INTERVAL_DAY_TO_SECOND: + // return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, + // IS_DISTINCT_FROM, + // IS_NOT_DISTINCT_FROM); + default: + return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, IS_NOT_DISTINCT_FROM); + } + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); + + private final String textRepresentation; + + PrestoBinaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoUnaryArithmeticOperator implements BinaryOperatorNode.Operator { + MINUS("-"); + + private final String textRepresentation; + + PrestoUnaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + private enum Expression { + BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC + } + + @Override + public PrestoTypedExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public PrestoExpression generateBooleanExpression() { + return generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), + randomly.getInteger(0, maxDepth)); + } + + @Override + public PrestoSelect generateSelect() { + return new PrestoSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new PrestoTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(PrestoSelect select, PrestoExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + PrestoFunctionNode aggr = new PrestoFunctionNode<>( + List.of(new PrestoColumnReference(new PrestoColumn("*", + new PrestoCompositeDataType(PrestoDataType.INT, 0, 0), false, false))), + PrestoAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + + } else { + List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(PrestoSelect select, PrestoExpression whereCondition) { + PrestoExpression asText = new PrestoPostfixText( + + new PrestoCastFunction( + new PrestoPostfixText(whereCondition, + " IS NOT NULL AND " + PrestoToStringVisitor.asString(whereCondition)), + new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), + "as count"); + + select.setFetchColumns(List.of(asText)); + select.setWhereClause(null); + return "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (Randomly.getBoolean()) { + return List.of(new PrestoColumnReference(new PrestoColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java new file mode 100644 index 000000000..3c197f0b9 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.presto.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public final class PrestoUpdateGenerator extends AbstractUpdateGenerator { + + private final PrestoGlobalState globalState; + private PrestoTypedExpressionGenerator gen; + + private PrestoUpdateGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canonicalizeString = false; + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + return new PrestoUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + PrestoErrors.addInsertErrors(errors); + } + + @Override + protected void updateValue(PrestoColumn column) { + PrestoExpression expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(column.getType()); + PrestoErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(column.getType()); + } + sb.append(PrestoToStringVisitor.asString(expr)); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoViewGenerator.java b/src/sqlancer/presto/gen/PrestoViewGenerator.java new file mode 100644 index 000000000..72130cf82 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoViewGenerator.java @@ -0,0 +1,36 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; + +public final class PrestoViewGenerator { + + private PrestoViewGenerator() { + } + + public static SQLQueryAdapter generate(PrestoGlobalState globalState) { + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(PrestoToStringVisitor.asString(PrestoRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + PrestoErrors.addExpressionErrors(errors); + PrestoErrors.addGroupByErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..7ab23286a --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java @@ -0,0 +1,203 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoAggregateFunction; +import sqlancer.presto.ast.PrestoAlias; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionNode; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoUnaryPostfixOperation; +import sqlancer.presto.ast.PrestoUnaryPrefixOperation; + +public class PrestoQueryPartitioningAggregateTester extends PrestoQueryPartitioningBase + implements TestOracle { + + private String firstResult; + private String firstResultType; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public PrestoQueryPartitioningAggregateTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + PrestoErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + PrestoAggregateFunction aggregateFunction = Randomly.fromOptions(PrestoAggregateFunction.MAX, + PrestoAggregateFunction.MIN, PrestoAggregateFunction.SUM, PrestoAggregateFunction.COUNT, + PrestoAggregateFunction.AVG/* , PrestoAggregateFunction.STDDEV_POP */); + List aggregateArgs = gen.generateArgsForAggregate(aggregateFunction); + PrestoFunctionNode aggregate = new PrestoFunctionNode<>(aggregateArgs, + aggregateFunction); + select.setFetchColumns(List.of(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + originalQuery = PrestoToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + firstResultType = getAggregateResultType(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResultType.equals("VARBINARY") || firstResultType.equals("ARRAY(VARBINARY)") + || firstResultType.equals("ARRAY(ARRAY(VARBINARY))")) { + throw new IgnoreMeException(); + } + if (firstResult == null && secondResult != null) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } else if (firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(PrestoSelect select, + PrestoFunctionNode aggregate, List from) { + String metamorphicQuery; + PrestoExpression whereClause = gen.generatePredicate(); + PrestoExpression negatedClause = new PrestoUnaryPrefixOperation(whereClause, + PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT); + PrestoExpression notNullClause = new PrestoUnaryPostfixOperation(whereClause, + PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + PrestoSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + PrestoSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + PrestoSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += PrestoToStringVisitor.asString(leftSelect) + " UNION ALL " + + PrestoToStringVisitor.asString(middleSelect) + " UNION ALL " + + PrestoToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private String getAggregateResultType(String queryString) { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getType(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List mapped(PrestoFunctionNode aggregate) { + PrestoCastFunction count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(List.of(aggregate)); + case AVG: + PrestoFunctionNode sum = new PrestoFunctionNode<>(aggregate.getArgs(), + PrestoAggregateFunction.SUM); + count = new PrestoCastFunction(new PrestoFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.COUNT), + new PrestoCompositeDataType(PrestoDataType.FLOAT, 8, 0)); + return aliasArgs(Arrays.asList(sum, count)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (PrestoExpression expr : originalAggregateArgs) { + args.add(new PrestoAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(PrestoFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case AVG: + return "SUM(CAST(agg0 AS DOUBLE))/CAST(SUM(agg1) AS DOUBLE)"; + case COUNT: + return PrestoAggregateFunction.SUM + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private PrestoSelect getSelect(List aggregates, List from, + PrestoExpression whereClause, List joinList) { + PrestoSelect leftSelect = new PrestoSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java new file mode 100644 index 000000000..0e80d5c41 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java @@ -0,0 +1,88 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.ast.PrestoColumnReference; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoQueryPartitioningBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + PrestoSchema s; + PrestoTables targetTables; + PrestoTypedExpressionGenerator gen; + PrestoSelect select; + + public PrestoQueryPartitioningBase(PrestoGlobalState state) { + super(state); + PrestoErrors.addExpressionErrors(errors); + } + + public static String canonicalizeResultValue(String value) { + if (value == null) { + return null; + } + + // TODO: check this + switch (value) { + case "-0.0": + return "0.0"; + case "-0": + return "0"; + default: + } + + return value; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new PrestoTypedExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new PrestoSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableList)); + select.setWhereClause(null); + } + + List generateFetchColumns() { + List columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new PrestoColumnReference(new PrestoColumn("*", null, false, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..008e8f4db --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java @@ -0,0 +1,44 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; + +public class PrestoQueryPartitioningDistinctTester extends PrestoQueryPartitioningBase { + + public PrestoQueryPartitioningDistinctTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + if (Randomly.getBoolean()) { + select.setDistinct(false); + } + select.setWhereClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..6c3f4bf71 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java @@ -0,0 +1,51 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoColumnReference; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoQueryPartitioningGroupByTester extends PrestoQueryPartitioningBase { + + public PrestoQueryPartitioningGroupByTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(select.getFetchColumns()); + select.setWhereClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java new file mode 100644 index 000000000..b53bfc07f --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java @@ -0,0 +1,63 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoQueryPartitioningHavingTester extends PrestoQueryPartitioningBase + implements TestOracle { + + public PrestoQueryPartitioningHavingTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull())); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + protected PrestoExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List generateFetchColumns() { + return Collections.singletonList(gen.generateHavingClause()); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java new file mode 100644 index 000000000..1fb2b7018 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java @@ -0,0 +1,45 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; + + public PrestoQueryPartitioningWhereTester(PrestoGlobalState state) { + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) + .with(PrestoErrors.getGroupByErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); + } + + @Override + public void check() throws SQLException { + oracle.check(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } +} diff --git a/src/sqlancer/questdb/QuestDBBugs.java b/src/sqlancer/questdb/QuestDBBugs.java new file mode 100644 index 000000000..1bd565823 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBBugs.java @@ -0,0 +1,7 @@ +package sqlancer.questdb; + +public final class QuestDBBugs { + + private QuestDBBugs() { + } +} diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java new file mode 100644 index 000000000..213f06ad4 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -0,0 +1,56 @@ +package sqlancer.questdb; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class QuestDBErrors { + + private QuestDBErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + // TODO (anxing) + errors.add("unexpected argument for function: "); + errors.add("unexpected token:"); // SELECT FROM multiple tables without WHERE/ JOIN clause + errors.add("boolean expression expected"); + errors.add("Column name expected"); + errors.add("too few arguments for 'in'"); + errors.add("cannot compare TIMESTAMP with type"); // WHERE column IN with nonTIMESTAMP arg + errors.add("constant expected"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getGroupByErrors() { + // TODO (anxing) + + return new ArrayList<>(); + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + // TODO (anxing) + errors.add("Invalid column"); + errors.add("inconvertible types:"); + errors.add("inconvertible value:"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java new file mode 100644 index 000000000..82cf55bd9 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -0,0 +1,39 @@ +package sqlancer.questdb; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(separators = "=", commandDescription = "QuestDB (default port: " + QuestDBOptions.DEFAULT_PORT + + " default host: " + QuestDBOptions.DEFAULT_HOST + ")") +public class QuestDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 8812; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(QuestDBOracleFactory.WHERE); + + @Parameter(names = "--username", description = "The user name used to log into QuestDB") + private String userName = "admin"; // NOPMD + + @Parameter(names = "--password", description = "The password used to log into QuestDB") + private String password = "quest"; // NOPMD + + @Override + public List getTestOracleFactory() { + return oracle; + } + + public String getUserName() { + return userName; + } + + public String getPassword() { + return password; + } + +} diff --git a/src/sqlancer/questdb/QuestDBOracleFactory.java b/src/sqlancer/questdb/QuestDBOracleFactory.java new file mode 100644 index 000000000..52c727278 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBOracleFactory.java @@ -0,0 +1,18 @@ +package sqlancer.questdb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.questdb.test.QuestDBQueryPartitioningWhereTester; + +public enum QuestDBOracleFactory implements OracleFactory { + // TODO (anxing): implement test oracles + WHERE { + @Override + public TestOracle create(QuestDBProvider.QuestDBGlobalState globalState) + throws SQLException { + return new QuestDBQueryPartitioningWhereTester(globalState); + } + } +} diff --git a/src/sqlancer/questdb/QuestDBProvider.java b/src/sqlancer/questdb/QuestDBProvider.java new file mode 100644 index 000000000..79c1554be --- /dev/null +++ b/src/sqlancer/questdb/QuestDBProvider.java @@ -0,0 +1,154 @@ +package sqlancer.questdb; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.gen.QuestDBAlterIndexGenerator; +import sqlancer.questdb.gen.QuestDBInsertGenerator; +import sqlancer.questdb.gen.QuestDBTableGenerator; +import sqlancer.questdb.gen.QuestDBTruncateGenerator; + +@AutoService(DatabaseProvider.class) +public class QuestDBProvider extends SQLProviderAdapter { + public QuestDBProvider() { + super(QuestDBGlobalState.class, QuestDBOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(QuestDBInsertGenerator::getQuery), // + ALTER_INDEX(QuestDBAlterIndexGenerator::getQuery), // + TRUNCATE(QuestDBTruncateGenerator::generate); // + // TODO (anxing): maybe implement these later + // UPDATE(QuestDBUpdateGenerator::getQuery), // + // CREATE_VIEW(QuestDBViewGenerator::generate), // + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(QuestDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(QuestDBGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case ALTER_INDEX: + return r.getInteger(0, 3); + case TRUNCATE: + return r.getInteger(0, 5); + default: + throw new AssertionError("Unknown action: " + a); + } + } + + public static class QuestDBGlobalState extends SQLGlobalState { + + @Override + protected QuestDBSchema readSchema() throws SQLException { + return QuestDBSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(QuestDBGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new QuestDBTableGenerator().getQuery(globalState, null); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + QuestDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(QuestDBGlobalState globalState) throws Exception { + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = QuestDBOptions.DEFAULT_HOST; + } + if (port == sqlancer.MainOptions.NO_SET_PORT) { + port = QuestDBOptions.DEFAULT_PORT; + } + // TODO(anxing): maybe not hardcode here... + String databaseName = "qdb"; + String tableName = "sqlancer_test"; + String url = String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); + // use QuestDB default username & password for Postgres JDBC + Properties properties = new Properties(); + properties.setProperty("user", globalState.getDbmsSpecificOptions().getUserName()); + properties.setProperty("password", globalState.getDbmsSpecificOptions().getPassword()); + properties.setProperty("sslmode", "disable"); + + Connection con = DriverManager.getConnection(url, properties); + // QuestDB cannot create or drop `DATABASE`, can only create or drop `TABLE` + globalState.getState().logStatement("DROP TABLE IF EXISTS " + tableName + " CASCADE"); + SQLQueryAdapter createTableCommand = new QuestDBTableGenerator().getQuery(globalState, tableName); + globalState.getState().logStatement(createTableCommand); + globalState.getState().logStatement("DROP TABLE IF EXISTS " + tableName); + + try (Statement s = con.createStatement()) { + s.execute("DROP TABLE IF EXISTS " + tableName); + } + // TODO(anxing): Drop all previous tables in db + // List tableNames = + // globalState.getSchema().getDatabaseTables().stream().map(AbstractTable::getName).collect(Collectors.toList()); + // for (String tName : tableNames) { + // try (Statement s = con.createStatement()) { + // String query = "DROP TABLE IF EXISTS " + tName; + // globalState.getState().logStatement(query); + // s.execute(query); + // } + // } + try (Statement s = con.createStatement()) { + s.execute(createTableCommand.getQueryString()); + } + // drop test table + try (Statement s = con.createStatement()) { + s.execute("DROP TABLE IF EXISTS " + tableName); + } + con.close(); + con = DriverManager.getConnection(url, properties); + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "questdb"; + } + +} diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java new file mode 100644 index 000000000..55ee01aab --- /dev/null +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -0,0 +1,314 @@ +package sqlancer.questdb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public class QuestDBSchema extends AbstractSchema { + + public enum QuestDBDataType { + + BOOLEAN, // CHAR, + /* STRING, */ + INT, FLOAT, SYMBOL, + // DATE, TIMESTAMP, + /* GEOHASH, */ + NULL; + + public static QuestDBDataType getRandomWithoutNull() { + QuestDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == QuestDBDataType.NULL); + return dt; + } + + } + + public static class QuestDBCompositeDataType { + + private final QuestDBDataType dataType; + + private final int size; + + private final boolean isNullable; + + public QuestDBCompositeDataType(QuestDBDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + + switch (dataType) { + case INT: + switch (size) { + case 1: + case 2: + isNullable = false; + break; + default: + isNullable = true; + break; + } + break; + case BOOLEAN: + isNullable = false; + break; + case SYMBOL: + isNullable = true; + break; + default: + isNullable = true; + } + } + + public QuestDBDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public boolean isNullable() { + return isNullable; + } + + public static QuestDBCompositeDataType getRandomWithoutNull() { + QuestDBDataType type = QuestDBDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8, 32); + break; + case BOOLEAN: + // case CHAR: + // case DATE: + // case TIMESTAMP: + size = 0; + break; + case SYMBOL: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new QuestDBCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 1: + return Randomly.fromOptions("BYTE"); + case 2: + return Randomly.fromOptions("SHORT"); + case 4: + return Randomly.fromOptions("INT"); + default: + throw new AssertionError(size); + } + // case CHAR: + // return "CHAR"; + case FLOAT: + switch (size) { + case 4: + return Randomly.fromOptions("FLOAT"); + case 8: + return Randomly.fromOptions(/* "DOUBLE", */"LONG"); + case 32: + return Randomly.fromOptions("LONG256"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN"); + case SYMBOL: + return "SYMBOL"; + // case TIMESTAMP: + // return Randomly.fromOptions("TIMESTAMP"); + // case DATE: + // return Randomly.fromOptions("DATE"); + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class QuestDBColumn extends AbstractTableColumn { + private final boolean isIndexed; + private final boolean isNullable; + + public QuestDBColumn(String name, QuestDBCompositeDataType columnType, boolean isIndexed) { + super(name, null, columnType); + this.isIndexed = isIndexed; + this.isNullable = columnType == null || columnType.isNullable(); + } + + public boolean isIndexed() { + return isIndexed; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class QuestDBTables extends AbstractTables { + public static final Set RESERVED_TABLES = new HashSet<>( + Arrays.asList("sys.column_versions_purge_log", "telemetry_config", "telemetry", "sys.telemetry_wal")); + + public QuestDBTables(List tables) { + super(tables); + } + } + + public QuestDBSchema(List databaseTables) { + super(databaseTables); + } + + public QuestDBTables getRandomTableNonEmptyTables() { + return new QuestDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + private static QuestDBCompositeDataType getColumnType(String typeString) { + QuestDBDataType primitiveType; + int size = -1; + + switch (typeString) { + case "INT": + primitiveType = QuestDBDataType.INT; + size = 4; + break; + // case "CHAR": + // primitiveType = QuestDBDataType.CHAR; + // break; + case "FLOAT": + primitiveType = QuestDBDataType.FLOAT; + size = 4; + break; + case "LONG": + primitiveType = QuestDBDataType.FLOAT; + size = 8; + break; + case "LONG256": + primitiveType = QuestDBDataType.FLOAT; + size = 32; + break; + case "BOOLEAN": + primitiveType = QuestDBDataType.BOOLEAN; + break; + // case "DATE": + // primitiveType = QuestDBDataType.DATE; + // break; + // case "TIMESTAMP": + // primitiveType = QuestDBDataType.TIMESTAMP; + // break; + case "BYTE": + primitiveType = QuestDBDataType.INT; + size = 1; + break; + case "SHORT": + primitiveType = QuestDBDataType.INT; + size = 2; + break; + case "SYMBOL": + primitiveType = QuestDBDataType.SYMBOL; + break; + case "NULL": + primitiveType = QuestDBDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + return new QuestDBCompositeDataType(primitiveType, size); + } + + public static class QuestDBTable extends AbstractRelationalTable { + + public QuestDBTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static QuestDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = matchesViewName(tableName); + QuestDBTable t = new QuestDBTable(tableName, databaseColumns, isView); + for (QuestDBColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new QuestDBSchema(databaseTables); + } + + protected static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES;")) { + while (rs.next()) { + String tName = rs.getString("table"); + // exclude reserved tables for testing + if (!QuestDBTables.RESERVED_TABLES.contains(tName)) { + tableNames.add(tName); + } + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("SHOW COLUMNS FROM %s;", tableName))) { + while (rs.next()) { + String columnName = rs.getString("column"); + String dataType = rs.getString("type"); + boolean isIndexed = rs.getString("indexed").contains("true"); + QuestDBColumn c = new QuestDBColumn(columnName, getColumnType(dataType), isIndexed); + columns.add(c); + } + } + } + return columns; + } + +} diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java new file mode 100644 index 000000000..9363b30b1 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -0,0 +1,66 @@ +package sqlancer.questdb; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.questdb.ast.QuestDBConstant; +import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBSelect; + +public class QuestDBToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(QuestDBExpression expr) { + if (expr instanceof QuestDBConstant) { + visit((QuestDBConstant) expr); + } else if (expr instanceof QuestDBSelect) { + visit((QuestDBSelect) expr); + } else { // TODO: maybe implement QuestDBJoin + throw new AssertionError("Unknown class: " + expr.getClass()); + } + } + + private void visit(QuestDBConstant constant) { + sb.append(constant.toString()); + } + + private void visit(QuestDBSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + // if (!select.getGroupByExpressions().isEmpty()) { + // sb.append(" GROUP BY "); + // visit(select.getGroupByExpressions()); + // } + // if (select.getHavingClause() != null) { + // sb.append(" HAVING "); + // visit(select.getHavingClause()); + // } + // if (!select.getOrderByClauses().isEmpty()) { + // sb.append(" ORDER BY "); + // visit(select.getOrderByClauses()); + // } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + } + + public static String asString(QuestDBExpression expr) { + QuestDBToStringVisitor visitor = new QuestDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java b/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java new file mode 100644 index 000000000..1b69b2c31 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class QuestDBBinaryOperation extends NewBinaryOperatorNode implements QuestDBExpression { + public QuestDBBinaryOperation(QuestDBExpression left, QuestDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBColumnReference.java b/src/sqlancer/questdb/ast/QuestDBColumnReference.java new file mode 100644 index 000000000..d627538a0 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.questdb.QuestDBSchema; + +public class QuestDBColumnReference extends ColumnReferenceNode + implements QuestDBExpression { + public QuestDBColumnReference(QuestDBSchema.QuestDBColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java new file mode 100644 index 000000000..f2e65fbc0 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -0,0 +1,111 @@ +package sqlancer.questdb.ast; + +public class QuestDBConstant implements QuestDBExpression { + private QuestDBConstant() { + } + + public static class QuestDBNullConstant extends QuestDBConstant { + @Override + public String toString() { + return "NULL"; + } + } + + public static class QuestDBIntConstant extends QuestDBConstant { + private final long value; + + public QuestDBIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + } + + public static class QuestDBBooleanConstant extends QuestDBConstant { + private final boolean value; + + public QuestDBBooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public boolean getValue() { + return value; + } + } + + public static class QuestDBSymbolConstant extends QuestDBConstant { + private final String value; + + public QuestDBSymbolConstant(String value) { + this.value = value; + } + + @Override + public String toString() { + if (value.equals("")) { + return "NULL"; + } + return "'" + value + "'"; + } + + public String getValue() { + return value; + } + } + + public static QuestDBExpression createIntConstant(long val) { + return new QuestDBIntConstant(val); + } + + public static class QuestDBDoubleConstant extends QuestDBConstant { + + private final double value; + + public QuestDBDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "cast('Infinity' as double)"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "cast('-Infinity' as double)"; + } + return String.valueOf(value); + } + + } + + public static QuestDBExpression createBooleanConstant(boolean val) { + return new QuestDBBooleanConstant(val); + } + + public static QuestDBExpression createNullConstant() { + return new QuestDBNullConstant(); + } + + public static QuestDBExpression createFloatConstant(double val) { + return new QuestDBDoubleConstant(val); + } + + public static QuestDBExpression createSymbolConstant(String val) { + return new QuestDBSymbolConstant(val); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBExpression.java b/src/sqlancer/questdb/ast/QuestDBExpression.java new file mode 100644 index 000000000..c2c7f1d74 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.questdb.ast; + +public interface QuestDBExpression { +} diff --git a/src/sqlancer/questdb/ast/QuestDBInOperation.java b/src/sqlancer/questdb/ast/QuestDBInOperation.java new file mode 100644 index 000000000..ee4ba2b59 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBInOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class QuestDBInOperation extends NewInOperatorNode implements QuestDBExpression { + public QuestDBInOperation(QuestDBExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBSelect.java b/src/sqlancer/questdb/ast/QuestDBSelect.java new file mode 100644 index 000000000..0684b4082 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBSelect.java @@ -0,0 +1,15 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.SelectBase; + +public class QuestDBSelect extends SelectBase implements QuestDBExpression { + private boolean isDistinct; + + public void setDistinct(boolean distinct) { + isDistinct = distinct; + } + + public boolean isDistinct() { + return isDistinct; + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBTableReference.java b/src/sqlancer/questdb/ast/QuestDBTableReference.java new file mode 100644 index 000000000..8e4d387da --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.questdb.QuestDBSchema; + +public class QuestDBTableReference extends TableReferenceNode + implements QuestDBExpression { + public QuestDBTableReference(QuestDBSchema.QuestDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java b/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java new file mode 100644 index 000000000..0301ba60a --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class QuestDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPostfixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java b/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java new file mode 100644 index 000000000..8488a0de8 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class QuestDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPrefixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java new file mode 100644 index 000000000..85103738f --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java @@ -0,0 +1,61 @@ +package sqlancer.questdb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBDataType; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public final class QuestDBAlterIndexGenerator { + private QuestDBAlterIndexGenerator() { + } + + enum Action { + ADD_INDEX, DROP_INDEX + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add(" does not have a column with name \"rowid\""); + errors.add("Table does not contain column rowid referenced in alter statement"); + errors.add("cannot create index"); + errors.add("Index flag is only supported for SYMBOL"); + errors.add("Invalid column: "); + + StringBuilder sb = new StringBuilder("ALTER TABLE "); + + QuestDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + + sb.append("ALTER COLUMN "); + + // We should always choose column with SYMBOL type + QuestDBColumn columnWithSymbolType = table + .getRandomColumnOrBailout(c -> c.getType().getPrimitiveDataType() == QuestDBDataType.SYMBOL); + + String columnName = columnWithSymbolType.getName(); + + sb.append(columnName); + sb.append(" "); + + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_INDEX: + sb.append("ADD INDEX"); + errors.add("already exists!"); + + break; + case DROP_INDEX: + sb.append("DROP INDEX"); + errors.add("Column is not indexed"); + break; + default: + throw new AssertionError("Unkown action:" + action); + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java new file mode 100644 index 000000000..cbb19e705 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -0,0 +1,203 @@ +package sqlancer.questdb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.Randomly.StringGenerationStrategy; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBDataType; +import sqlancer.questdb.ast.QuestDBBinaryOperation; +import sqlancer.questdb.ast.QuestDBColumnReference; +import sqlancer.questdb.ast.QuestDBConstant; +import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBInOperation; +import sqlancer.questdb.ast.QuestDBUnaryPostfixOperation; +import sqlancer.questdb.ast.QuestDBUnaryPrefixOperation; + +public class QuestDBExpressionGenerator extends UntypedExpressionGenerator { + + private final QuestDBGlobalState globalState; + + public QuestDBExpressionGenerator(QuestDBGlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, IN + } + + @Override + public QuestDBExpression negatePredicate(QuestDBExpression predicate) { + return new QuestDBUnaryPrefixOperation(predicate, QuestDBUnaryPrefixOperator.NOT); + } + + @Override + public QuestDBExpression isNull(QuestDBExpression expr) { + return new QuestDBUnaryPostfixOperation(expr, QuestDBUnaryPostfixOperator.IS_NULL); + } + + @Override + public QuestDBExpression generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return QuestDBConstant.createNullConstant(); + } + QuestDBDataType type = QuestDBDataType.getRandomWithoutNull(); + switch (type) { + case INT: + return QuestDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + case BOOLEAN: + return QuestDBConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return QuestDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + case SYMBOL: + StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; + return QuestDBConstant.createSymbolConstant(strategy.getString(globalState.getRandomly())); + // case CHAR: + // case DATE: + // case TIMESTAMP: + // throw new IgnoreMeException(); + default: + throw new AssertionError("Unknown type: " + type); + } + } + + @Override + protected QuestDBExpression generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + + switch (expr) { + case UNARY_PREFIX: + return new QuestDBUnaryPrefixOperation(generateExpression(depth + 1), + QuestDBUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new QuestDBUnaryPostfixOperation(generateExpression(depth + 1), + QuestDBUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryComparisonOperator.getRandom()); + case BINARY_ARITHMETIC: + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryArithmeticOperator.getRandom()); + case BINARY_LOGICAL: + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryLogicalOperator.getRandom()); + case IN: + return new QuestDBInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + default: + throw new AssertionError("Expression generation failed, depth=" + depth); + } + } + + @Override + protected QuestDBExpression generateColumn() { + QuestDBColumn column = Randomly.fromList(columns); + return new QuestDBColumnReference(column); + } + + public enum QuestDBUnaryPostfixOperator implements Operator { + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + QuestDBUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static QuestDBUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum QuestDBUnaryPrefixOperator implements Operator { + + NOT("NOT"); + + private String textRepr; + + QuestDBUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static QuestDBUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum QuestDBBinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum QuestDBBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER_THAN(">"), GREATER_THAN_EQUALS(">="), LESS_THAN("<"), SMALLER_THAN_EQUALS("<="), + NOT_EQUALS("!="), REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + + private String textRepr; + + QuestDBBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum QuestDBBinaryArithmeticOperator implements Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"); // , LSHIFT("<<"), + // RSHIFT(">>"); + + private String textRepr; + + QuestDBBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java new file mode 100644 index 000000000..41ba72406 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.questdb.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; +import sqlancer.questdb.QuestDBToStringVisitor; + +public class QuestDBInsertGenerator extends AbstractInsertGenerator { + + private final QuestDBGlobalState globalState; + + public QuestDBInsertGenerator(QuestDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + return new QuestDBInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + QuestDBTable table = globalState.getSchema().getRandomTable(); + List columns = table.getRandomNonEmptyColumnSubset(); + buildInsertInto(table.getName(), columns); + QuestDBErrors.addInsertErrors(errors); + } + + @Override + protected void insertColumns(List columns) { + sb.append("("); + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + insertValue(columns.get(nrColumn)); + } + sb.append(")"); + } + + @Override + protected void insertValue(QuestDBColumn questDBColumn) { + sb.append(QuestDBToStringVisitor.asString(new QuestDBExpressionGenerator(globalState).generateConstant())); + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java new file mode 100644 index 000000000..d4b17e76b --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -0,0 +1,50 @@ +package sqlancer.questdb.gen; + +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; + +public class QuestDBTableGenerator extends AbstractTableGenerator { + + private QuestDBGlobalState globalState; + private String tableName; + + public QuestDBTableGenerator() { + this.canAffectSchema = true; + } + + public SQLQueryAdapter getQuery(QuestDBGlobalState globalState, @Nullable String tableName) { + this.globalState = globalState; + this.tableName = tableName; + return getStatement(); + } + + @Override + public void buildStatement() { + String name = tableName; + if (name == null) { + name = globalState.getSchema().getFreeTableName(); + } + appendCreateTable(name, Randomly.getBoolean()); + appendColumnDefinitions(getNewColumns()); + sb.append(";"); + errors.add("table already exists"); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + QuestDBCompositeDataType columnType = QuestDBCompositeDataType.getRandomWithoutNull(); + columns.add(new QuestDBColumn(columnName, columnType, false)); + } + return columns; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java b/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java new file mode 100644 index 000000000..19c48f9f7 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java @@ -0,0 +1,22 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public final class QuestDBTruncateGenerator { + private QuestDBTruncateGenerator() { + + } + + public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { + StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); + ExpectedErrors errors = new ExpectedErrors(); + QuestDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + QuestDBErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java b/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java new file mode 100644 index 000000000..cc7fba1df --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java @@ -0,0 +1,16 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBUpdateGenerator { + + private QuestDBUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + // TODO + return null; + } + +} diff --git a/src/sqlancer/questdb/gen/QuestDBViewGenerator.java b/src/sqlancer/questdb/gen/QuestDBViewGenerator.java new file mode 100644 index 000000000..0ff4d232b --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBViewGenerator.java @@ -0,0 +1,14 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBViewGenerator { + private QuestDBViewGenerator() { + } + + public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { + // TODO + return null; + } +} diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java new file mode 100644 index 000000000..0716e9ccc --- /dev/null +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -0,0 +1,71 @@ +package sqlancer.questdb.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; +import sqlancer.questdb.ast.QuestDBColumnReference; +import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBSelect; +import sqlancer.questdb.ast.QuestDBTableReference; +import sqlancer.questdb.gen.QuestDBExpressionGenerator; + +public class QuestDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + QuestDBSchema s; + QuestDBTable targetTable; + QuestDBExpressionGenerator gen; + QuestDBSelect select; + + protected QuestDBQueryPartitioningBase(QuestDBGlobalState state) { + super(state); + QuestDBErrors.addExpressionErrors(errors); + } + + List generateFetchColumns() { + List columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new QuestDBColumnReference(new QuestDBColumn("*", null, false))); + } else { + columns = Randomly.nonEmptySubset(targetTable.getColumns()).stream().map(c -> new QuestDBColumnReference(c)) + .collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + // Only return one table instead of multiple tables, which is regarded as illegal by QuestDB + // e.g. "SELECT * FROM t0, t1;" + targetTable = s.getRandomTable(); + gen = new QuestDBExpressionGenerator(state).setColumns(targetTable.getColumns()); + initializeTernaryPredicateVariants(); + select = new QuestDBSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = new ArrayList<>(); + tables.add(targetTable); + List tableList = tables.stream().map(t -> new QuestDBTableReference(t)) + .collect(Collectors.toList()); + // Ignore JOINs for now + select.setFromList(new ArrayList<>(tableList)); + select.setWhereClause(null); + } +} diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..ed89e7136 --- /dev/null +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java @@ -0,0 +1,41 @@ +package sqlancer.questdb.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBToStringVisitor; + +public class QuestDBQueryPartitioningWhereTester extends QuestDBQueryPartitioningBase { + public QuestDBQueryPartitioningWhereTester(QuestDBGlobalState state) { + super(state); + QuestDBErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = QuestDBToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + // Ignore OrderBy for now + + select.setWhereClause(predicate); + String firstQueryString = QuestDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = QuestDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = QuestDBToStringVisitor.asString(select); + + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, false, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java new file mode 100644 index 000000000..a3a96061f --- /dev/null +++ b/src/sqlancer/spark/SparkErrors.java @@ -0,0 +1,65 @@ +package sqlancer.spark; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class SparkErrors { + + private SparkErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot resolve"); + errors.add("AnalysisException"); + errors.add("data type mismatch"); + errors.add("undefined function"); + errors.add("mismatched input"); + errors.add("due to data type mismatch"); + + // --- Invalid Literals + errors.add("The value of the typed literal"); + + errors.add("DATATYPE_MISMATCH"); + errors.add("cannot be cast to"); + + errors.add("Overflow"); + errors.add("Divide by zero"); // Common if spark.sql.ansi.enabled is true + errors.add("division by zero"); + + // --- Group By / Aggregation errors --- + errors.add("grouping expressions"); + errors.add("expression is neither present in the group by"); + errors.add("is not a valid grouping expression"); + errors.add("is not contained in either an aggregate function or the GROUP BY clause"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("not enough data columns"); + errors.add("cannot write to"); + errors.add("incompatible types"); + errors.add("too many data columns"); + errors.add("cannot be cast to"); + errors.add("Error running query"); + errors.add("The value of the typed literal"); + errors.add("Cannot safely cast"); // Found in logs: Decimal -> Date + errors.add("AnalysisException"); // Spark throws this for almost all insert failures + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} diff --git a/src/sqlancer/spark/SparkGlobalState.java b/src/sqlancer/spark/SparkGlobalState.java new file mode 100644 index 000000000..d78c737e5 --- /dev/null +++ b/src/sqlancer/spark/SparkGlobalState.java @@ -0,0 +1,11 @@ +package sqlancer.spark; + +import sqlancer.SQLGlobalState; + +public class SparkGlobalState extends SQLGlobalState { + + @Override + protected SparkSchema readSchema() throws Exception { + return SparkSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/spark/SparkOptions.java b/src/sqlancer/spark/SparkOptions.java new file mode 100644 index 000000000..7b347ceef --- /dev/null +++ b/src/sqlancer/spark/SparkOptions.java @@ -0,0 +1,43 @@ +package sqlancer.spark; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.spark.gen.SparkExpressionGenerator; + +@Parameters(separators = "=", commandDescription = "Spark SQL (default port: " + SparkOptions.DEFAULT_PORT + + ", default host: " + SparkOptions.DEFAULT_HOST + ")") +public class SparkOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 10000; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(SparkOracleFactory.TLPWhere); + + public enum SparkOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(SparkGlobalState globalState) throws SQLException { + SparkExpressionGenerator gen = new SparkExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SparkErrors.getExpressionErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }; + } + + @Override + public List getTestOracleFactory() { + return oracle; + } +} diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java new file mode 100644 index 000000000..817a92471 --- /dev/null +++ b/src/sqlancer/spark/SparkProvider.java @@ -0,0 +1,123 @@ +package sqlancer.spark; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.spark.gen.SparkInsertGenerator; +import sqlancer.spark.gen.SparkTableGenerator; + +@AutoService(DatabaseProvider.class) +public class SparkProvider extends SQLProviderAdapter { + + public SparkProvider() { + super(SparkGlobalState.class, SparkOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(SparkInsertGenerator::getQuery); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(SparkGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(SparkGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(SparkGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + String tableName = globalState.getSchema().getFreeTableName(); + SQLQueryAdapter qt = SparkTableGenerator.generate(globalState, tableName); + success = globalState.executeStatement(qt); + } while (!success); + } + + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + SparkProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + + if (host == null) { + host = SparkOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = SparkOptions.DEFAULT_PORT; + } + + String databaseName = globalState.getDatabaseName(); + + // Spark uses the Hive driver for JDBC usually + String url = String.format("jdbc:hive2://%s:%d/%s", host, port, "default"); + + // Connect to default to create the fuzzing DB + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + con.close(); + + // Connect to the specific fuzzing DB + con = DriverManager.getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName), username, + password); + try (Statement s = con.createStatement()) { + // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the + // generator loves to do. + s.execute("SET spark.sql.ansi.enabled=false"); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "spark"; + } +} diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java new file mode 100644 index 000000000..9b3666916 --- /dev/null +++ b/src/sqlancer/spark/SparkSchema.java @@ -0,0 +1,122 @@ +package sqlancer.spark; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.spark.SparkSchema.SparkTable; + +public class SparkSchema extends AbstractSchema { + + public enum SparkDataType { + STRING, INTEGER, DOUBLE, BOOLEAN, TIMESTAMP, DATE; + + public static SparkDataType getRandomType() { + return Randomly.fromList(Arrays.asList(values())); + } + } + + public static class SparkColumn extends AbstractTableColumn { + public SparkColumn(String name, SparkTable table, SparkDataType type) { + super(name, table, type); + } + } + + public static class SparkTables extends AbstractTables { + public SparkTables(List tables) { + super(tables); + } + } + + public static class SparkTable extends AbstractRelationalTable { + public SparkTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + } + + public SparkSchema(List databaseTables) { + super(databaseTables); + } + + public static SparkSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + boolean isView = matchesViewName(tableName); + SparkTable t = new SparkTable(tableName, databaseColumns, isView); + for (SparkColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new SparkSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet tableRs = s.executeQuery("SHOW TABLES"); + while (tableRs.next()) { + // Spark SHOW TABLES output: database, tableName, isTemporary + String tableName = tableRs.getString("tableName"); + tableNames.add(tableName); + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("DESCRIBE %s", tableName))) { + while (rs.next()) { + String columnName = rs.getString("col_name"); + String dataType = rs.getString("data_type"); + // Filter out Spark partition info or comments usually at bottom of describe + if (columnName.startsWith("#") || columnName.isEmpty()) { + continue; + } + + columns.add(new SparkColumn(columnName, null, getColumnType(dataType))); + } + } + } + return columns; + } + + private static SparkDataType getColumnType(String typeString) { + String upper = typeString.toUpperCase(); + if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) { + return SparkDataType.STRING; + } + if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) { + return SparkDataType.INTEGER; + } + if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) { + return SparkDataType.DOUBLE; + } + if (upper.startsWith("BOOLEAN")) { + return SparkDataType.BOOLEAN; + } + if (upper.startsWith("TIMESTAMP")) { + return SparkDataType.TIMESTAMP; + } + if (upper.startsWith("DATE")) { + return SparkDataType.DATE; + } + return SparkDataType.STRING; // Fallback + } + +} diff --git a/src/sqlancer/spark/SparkToStringVisitor.java b/src/sqlancer/spark/SparkToStringVisitor.java new file mode 100644 index 000000000..0777c86a6 --- /dev/null +++ b/src/sqlancer/spark/SparkToStringVisitor.java @@ -0,0 +1,121 @@ +package sqlancer.spark; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.spark.ast.SparkCastOperation; +import sqlancer.spark.ast.SparkConstant; +import sqlancer.spark.ast.SparkExpression; +import sqlancer.spark.ast.SparkJoin; +import sqlancer.spark.ast.SparkSelect; + +public class SparkToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(SparkExpression expr) { + if (expr instanceof SparkConstant) { + visit((SparkConstant) expr); + } else if (expr instanceof SparkSelect) { + visit((SparkSelect) expr); + } else if (expr instanceof SparkJoin) { + visit((SparkJoin) expr); + } else if (expr instanceof SparkCastOperation) { + visit((SparkCastOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(SparkConstant constant) { + sb.append(constant.toString()); + } + + private void visit(SparkSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + // Spark supports OFFSET, though strictly usually with LIMIT or in newer + // versions + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(SparkJoin join) { + switch (join.getJoinType()) { + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + // Spark also supports LEFT ANTI, which Hive might lack in some older versions + case LEFT_ANTI: + sb.append(" LEFT ANTI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException("Join type not supported in Spark visitor: " + join.getJoinType()); + } + visit((TableReferenceNode) join.getRightTable()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + private void visit(SparkCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(")"); + } + + public static String asString(SparkExpression expr) { + SparkToStringVisitor visitor = new SparkToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/spark/ast/SparkBetweenOperation.java b/src/sqlancer/spark/ast/SparkBetweenOperation.java new file mode 100644 index 000000000..59297ba8f --- /dev/null +++ b/src/sqlancer/spark/ast/SparkBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class SparkBetweenOperation extends NewBetweenOperatorNode implements SparkExpression { + + public SparkBetweenOperation(SparkExpression left, SparkExpression middle, SparkExpression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/spark/ast/SparkBinaryOperation.java b/src/sqlancer/spark/ast/SparkBinaryOperation.java new file mode 100644 index 000000000..ef3347018 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class SparkBinaryOperation extends NewBinaryOperatorNode implements SparkExpression { + + public SparkBinaryOperation(SparkExpression left, SparkExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/spark/ast/SparkCaseOperation.java b/src/sqlancer/spark/ast/SparkCaseOperation.java new file mode 100644 index 000000000..995fd7f52 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkCaseOperation.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class SparkCaseOperation extends NewCaseOperatorNode implements SparkExpression { + + public SparkCaseOperation(SparkExpression switchCondition, List conditions, + List expressions, SparkExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/spark/ast/SparkCastOperation.java b/src/sqlancer/spark/ast/SparkCastOperation.java new file mode 100644 index 000000000..547551285 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkCastOperation.java @@ -0,0 +1,25 @@ +package sqlancer.spark.ast; + +import sqlancer.spark.SparkSchema.SparkDataType; + +public class SparkCastOperation implements SparkExpression { + + private final SparkExpression expression; + private final SparkDataType type; + + public SparkCastOperation(SparkExpression expression, SparkDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public SparkExpression getExpression() { + return expression; + } + + public SparkDataType getType() { + return type; + } +} diff --git a/src/sqlancer/spark/ast/SparkColumnReference.java b/src/sqlancer/spark/ast/SparkColumnReference.java new file mode 100644 index 000000000..ccd1b7855 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.spark.SparkSchema.SparkColumn; + +public class SparkColumnReference extends ColumnReferenceNode implements SparkExpression { + + public SparkColumnReference(SparkColumn column) { + super(column); + } +} diff --git a/src/sqlancer/spark/ast/SparkConstant.java b/src/sqlancer/spark/ast/SparkConstant.java new file mode 100644 index 000000000..84a397624 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkConstant.java @@ -0,0 +1,194 @@ +package sqlancer.spark.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public abstract class SparkConstant implements SparkExpression { + + public boolean isNull() { + return false; + } + + public static class SparkNullConstant extends SparkConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public String toString() { + return "NULL"; + } + } + + public static class SparkIntConstant extends SparkConstant { + + private final long value; + + public SparkIntConstant(long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class SparkDoubleConstant extends SparkConstant { + + private final double value; + + public SparkDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "CAST('Infinity' AS DOUBLE)"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "CAST('-Infinity' AS DOUBLE)"; + } else if (Double.isNaN(value)) { + return "CAST('NaN' AS DOUBLE)"; + } + return String.valueOf(value); + } + } + + public static class SparkDecimalConstant extends SparkConstant { + + private final BigDecimal value; + + public SparkDecimalConstant(BigDecimal value) { + this.value = value; + } + + public BigDecimal getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class SparkTimestampConstant extends SparkConstant { + + private final String textRepr; + + public SparkTimestampConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // Spark prefers full timestamp + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + } + + public static class SparkDateConstant extends SparkConstant { + + private final String textRepr; + + public SparkDateConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + } + + public static class SparkStringConstant extends SparkConstant { + + private final String value; + + public SparkStringConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''").replace("\\", "\\\\") + "'"; + } + } + + public static class SparkBooleanConstant extends SparkConstant { + + private final boolean value; + + public SparkBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static SparkConstant createNullConstant() { + return new SparkNullConstant(); + } + + public static SparkConstant createIntConstant(long value) { + return new SparkIntConstant(value); + } + + public static SparkConstant createDoubleConstant(double value) { + return new SparkDoubleConstant(value); + } + + public static SparkConstant createDecimalConstant(BigDecimal value) { + return new SparkDecimalConstant(value); + } + + public static SparkConstant createTimestampConstant(long value) { + return new SparkTimestampConstant(value); + } + + public static SparkConstant createDateConstant(long value) { + return new SparkDateConstant(value); + } + + public static SparkConstant createStringConstant(String value) { + return new SparkStringConstant(value); + } + + public static SparkConstant createBooleanConstant(boolean value) { + return new SparkBooleanConstant(value); + } +} diff --git a/src/sqlancer/spark/ast/SparkExpression.java b/src/sqlancer/spark/ast/SparkExpression.java new file mode 100644 index 000000000..3872ccfda --- /dev/null +++ b/src/sqlancer/spark/ast/SparkExpression.java @@ -0,0 +1,7 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.spark.SparkSchema.SparkColumn; + +public interface SparkExpression extends Expression { +} diff --git a/src/sqlancer/spark/ast/SparkFunction.java b/src/sqlancer/spark/ast/SparkFunction.java new file mode 100644 index 000000000..d5740ee36 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkFunction.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class SparkFunction extends NewFunctionNode implements SparkExpression { + + public SparkFunction(List args, F func) { + super(args, func); + } + +} diff --git a/src/sqlancer/spark/ast/SparkInOperation.java b/src/sqlancer/spark/ast/SparkInOperation.java new file mode 100644 index 000000000..430d9b5c2 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class SparkInOperation extends NewInOperatorNode implements SparkExpression { + + public SparkInOperation(SparkExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/spark/ast/SparkJoin.java b/src/sqlancer/spark/ast/SparkJoin.java new file mode 100644 index 000000000..a59eaff48 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkJoin.java @@ -0,0 +1,46 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.Join; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; + +public class SparkJoin implements SparkExpression, Join { + + private final SparkTableReference leftTable; + private final SparkTableReference rightTable; + private final JoinType joinType; + private SparkExpression onClause; + + public enum JoinType { + INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, LEFT_ANTI, CROSS; + } + + public SparkJoin(SparkTableReference leftTable, SparkTableReference rightTable, JoinType joinType, + SparkExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onClause = onClause; + } + + public SparkTableReference getLeftTable() { + return leftTable; + } + + public SparkTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public SparkExpression getOnClause() { + return onClause; + } + + @Override + public void setOnClause(SparkExpression onClause) { + this.onClause = onClause; + } +} diff --git a/src/sqlancer/spark/ast/SparkOrderingTerm.java b/src/sqlancer/spark/ast/SparkOrderingTerm.java new file mode 100644 index 000000000..870c8239b --- /dev/null +++ b/src/sqlancer/spark/ast/SparkOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class SparkOrderingTerm extends NewOrderingTerm implements SparkExpression { + + public SparkOrderingTerm(SparkExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/spark/ast/SparkSelect.java b/src/sqlancer/spark/ast/SparkSelect.java new file mode 100644 index 000000000..8b59f5513 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkSelect.java @@ -0,0 +1,42 @@ +package sqlancer.spark.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkSelect extends SelectBase + implements Select, SparkExpression { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (SparkExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (SparkJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return SparkToStringVisitor.asString(this); + } + +} diff --git a/src/sqlancer/spark/ast/SparkTableReference.java b/src/sqlancer/spark/ast/SparkTableReference.java new file mode 100644 index 000000000..5bcbb5d03 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkTableReference.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.spark.SparkSchema; + +public class SparkTableReference extends TableReferenceNode + implements SparkExpression { + + public SparkTableReference(SparkSchema.SparkTable table) { + super(table); + } + +} diff --git a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java new file mode 100644 index 000000000..3dd9d28e2 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class SparkUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements SparkExpression { + + public SparkUnaryPostfixOperation(SparkExpression expr, Operator op) { + super(expr, op); + } + +} diff --git a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java new file mode 100644 index 000000000..5c1a8e4c6 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class SparkUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements SparkExpression { + + public SparkUnaryPrefixOperation(SparkExpression expr, Operator op) { + super(expr, op); + } + +} diff --git a/src/sqlancer/spark/gen/SparkExpressionGenerator.java b/src/sqlancer/spark/gen/SparkExpressionGenerator.java new file mode 100644 index 000000000..3708f314a --- /dev/null +++ b/src/sqlancer/spark/gen/SparkExpressionGenerator.java @@ -0,0 +1,336 @@ +package sqlancer.spark.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkDataType; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.ast.SparkBetweenOperation; +import sqlancer.spark.ast.SparkBinaryOperation; +import sqlancer.spark.ast.SparkCaseOperation; +import sqlancer.spark.ast.SparkCastOperation; +import sqlancer.spark.ast.SparkColumnReference; +import sqlancer.spark.ast.SparkConstant; +import sqlancer.spark.ast.SparkExpression; +import sqlancer.spark.ast.SparkFunction; +import sqlancer.spark.ast.SparkInOperation; +import sqlancer.spark.ast.SparkJoin; +import sqlancer.spark.ast.SparkOrderingTerm; +import sqlancer.spark.ast.SparkSelect; +import sqlancer.spark.ast.SparkTableReference; +import sqlancer.spark.ast.SparkUnaryPostfixOperation; +import sqlancer.spark.ast.SparkUnaryPrefixOperation; + +public class SparkExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { + + private final SparkGlobalState globalState; + private List tables; + + private enum Expression { + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, CAST, FUNC, BETWEEN, IN, + CASE; + } + + public SparkExpressionGenerator(SparkGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public SparkExpression negatePredicate(SparkExpression predicate) { + return new SparkUnaryPrefixOperation(predicate, SparkUnaryPrefixOperator.NOT); + } + + @Override + public SparkExpression isNull(SparkExpression expr) { + return new SparkUnaryPostfixOperation(expr, SparkUnaryPostfixOperator.IS_NULL); + } + + @Override + protected SparkExpression generateExpression(int depth) { + return generateExpressionInternal(depth); + } + + private SparkExpression generateExpressionInternal(int depth) throws AssertionError { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { + allowAggregates = false; // aggregate function calls cannot be nested + SparkAggregateFunction aggregate = SparkAggregateFunction.getRandom(); + return new SparkFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + + switch (expr) { + case UNARY_PREFIX: + return new SparkUnaryPrefixOperation(generateExpression(depth + 1), SparkUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new SparkUnaryPostfixOperation(generateExpression(depth + 1), SparkUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = SparkBinaryComparisonOperator.getRandom(); + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = SparkBinaryLogicalOperator.getRandom(); + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + SparkBinaryArithmeticOperator.getRandom()); + case CAST: + return new SparkCastOperation(generateExpression(depth + 1), SparkDataType.getRandomType()); + case FUNC: + SparkFunc func = SparkFunc.getRandom(); + return new SparkFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new SparkBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new SparkInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new SparkCaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(expr); + } + } + + @Override + public SparkExpression generateConstant() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return SparkConstant.createNullConstant(); + } + SparkDataType[] values = SparkDataType.values(); + SparkDataType constantType = Randomly.fromOptions(values); + switch (constantType) { + case STRING: + return SparkConstant.createStringConstant(globalState.getRandomly().getString()); + case INTEGER: + return SparkConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return SparkConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return SparkConstant.createBooleanConstant(Randomly.getBoolean()); + case TIMESTAMP: + return SparkConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case DATE: + return SparkConstant.createDateConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(constantType); + } + } + + @Override + protected SparkExpression generateColumn() { + SparkColumn column = Randomly.fromList(columns); + return new SparkColumnReference(column); + } + + @Override + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (SparkExpression curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new SparkOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + } + + @Override + public SparkExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + return this; + } + + @Override + public SparkExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public SparkSelect generateSelect() { + return new SparkSelect(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new SparkTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean allowAggregates) { + if (Randomly.getBoolean()) { + return List.of(new SparkColumnReference(new SparkColumn("*", null, null))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new SparkColumnReference(c)) + .collect(Collectors.toList()); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + public enum SparkUnaryPrefixOperator implements Operator { + NOT("NOT"), PLUS("+"), MINUS("-"), BITWISE_NOT("~"); + + private String textRepr; + + SparkUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkUnaryPostfixOperator implements Operator { + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + SparkUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), RLIKE("RLIKE"); + + private String textRepr; + + SparkBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryLogicalOperator implements Operator { + AND("AND"), OR("OR"); + + private String textRepr; + + SparkBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryArithmeticOperator implements Operator { + // Spark supports || for concat, and bitwise operators &, |, ^ + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), BITWISE_AND("&"), BITWISE_OR("|"), + BITWISE_XOR("^"); + + private String textRepr; + + SparkBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkAggregateFunction { + COUNT(1), SUM(1), AVG(1), MIN(1), MAX(1), VARIANCE(1), VAR_SAMP(1), STDDEV_POP(1), STDDEV_SAMP(1), COVAR_POP(2), + COVAR_SAMP(2), CORR(2); + + private int nrArgs; + + SparkAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static SparkAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + public enum SparkFunc { + ROUND(2), FLOOR(1), ABS(1), CEIL(1); + + private int nrArgs; + private boolean isVariadic; + + SparkFunc(int nrArgs) { + this(nrArgs, false); + } + + SparkFunc(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static SparkFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + } +} diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java new file mode 100644 index 000000000..c43584315 --- /dev/null +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -0,0 +1,46 @@ +package sqlancer.spark.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.spark.SparkErrors; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkInsertGenerator extends AbstractInsertGenerator { + + private final SparkGlobalState globalState; + private final SparkExpressionGenerator gen; + + public SparkInsertGenerator(SparkGlobalState globalState) { + this.globalState = globalState; + this.gen = new SparkExpressionGenerator(globalState); + this.canonicalizeString = false; + } + + public static SQLQueryAdapter getQuery(SparkGlobalState globalState) { + return new SparkInsertGenerator(globalState).getStatement(); + } + + @Override + protected void insertValue(SparkColumn column) { + sb.append(SparkToStringVisitor.asString(gen.generateConstant())); + } + + @Override + public void buildStatement() { + sb.append("INSERT INTO "); + SparkTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + + sb.append(" VALUES "); + + List columns = table.getColumns(); + insertColumns(columns); + + SparkErrors.addInsertErrors(errors); + } +} diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java new file mode 100644 index 000000000..2c26ea34c --- /dev/null +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -0,0 +1,95 @@ +package sqlancer.spark.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.gen.AbstractTableGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.spark.SparkErrors; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkDataType; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkTableGenerator extends AbstractTableGenerator { + + private enum ColumnConstraints { + NOT_NULL, DEFAULT + // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources + // (Parquet/ORC) + // without specific catalogs (like Delta/Iceberg), so we limit to constraints + // Spark SQL widely accepts. + } + + private final SparkGlobalState globalState; + private final String tableName; + private final SparkExpressionGenerator gen; + private final SparkTable table; + private final List columnsToBeAdded = new ArrayList<>(); + + public SparkTableGenerator(SparkGlobalState globalState, String tableName) { + this.tableName = tableName; + this.globalState = globalState; + this.table = new SparkTable(tableName, columnsToBeAdded, false); + this.gen = new SparkExpressionGenerator(globalState).setColumns(columnsToBeAdded); + this.canAffectSchema = true; + this.canonicalizeString = false; + } + + public static SQLQueryAdapter generate(SparkGlobalState globalState, String tableName) { + return new SparkTableGenerator(globalState, tableName).getStatement(); + } + + @Override + public void buildStatement() { + int columnCount = Randomly.smallNumber() + 1; + for (int i = 0; i < columnCount; i++) { + String columnName = DBMSCommon.createColumnName(i); + SparkDataType type = SparkSchema.SparkDataType.getRandomType(); + columnsToBeAdded.add(new SparkColumn(columnName, table, type)); + } + appendCreateTable(globalState.getDatabaseName() + "." + tableName); + sb.append(" "); + appendColumnDefinitions(columnsToBeAdded); + sb.append(" USING PARQUET"); + + // TODO: implement PARTITION BY clause + // TODO: implement CLUSTERED BY clauses + // TODO: implement ROW FORMAT and STORED AS clauses + // TODO: randomly add some predefined TABLEPROPERTIES + + SparkErrors.addExpressionErrors(errors); + } + + @Override + protected void appendColumnDefinition(SparkColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + appendColumnConstraint(); + } + + private void appendColumnConstraint() { + if (Randomly.getBoolean()) { + return; + } + + ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); + switch (constraint) { + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + sb.append(" DEFAULT "); + sb.append(SparkToStringVisitor.asString(gen.generateConstant())); + sb.append(" "); + break; + default: + throw new AssertionError(constraint); + } + } +} diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 69a9b8d9e..96e45c549 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -1,27 +1,38 @@ package sqlancer.sqlite3; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.List; -import java.util.Set; + +import sqlancer.common.query.ExpectedErrors; public final class SQLite3Errors { private SQLite3Errors() { } - public static void addDeleteErrors(List errors) { + public static List getDeleteErrors() { + ArrayList errors = new ArrayList<>(); + // DELETE trigger for a view/table to which colomns were added or deleted errors.add("columns but"); // trigger with on conflict clause errors.add("ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint"); + + return errors; + } + + public static void addDeleteErrors(ExpectedErrors errors) { + errors.addAll(getDeleteErrors()); } - public static void addExpectedExpressionErrors(Collection errors) { + public static List getExpectedExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("[SQLITE_BUSY] The database file is locked"); errors.add("FTS expression tree is too large"); errors.add("String or BLOB exceeds size limit"); errors.add("[SQLITE_ERROR] SQL error or missing database (integer overflow)"); - errors.add("second argument to likelihood() must be a constant between 0.0 and 1.0"); errors.add("ORDER BY term out of range"); errors.add("GROUP BY term out of range"); errors.add("not authorized"); // load_extension @@ -45,9 +56,11 @@ public static void addExpectedExpressionErrors(Collection errors) { errors.add("malformed JSON"); errors.add("JSON cannot hold BLOB values"); errors.add("JSON path error"); + errors.add("bad JSON path"); errors.add("json_insert() needs an odd number of arguments"); errors.add("json_object() labels must be TEXT"); errors.add("json_object() requires an even number of arguments"); + errors.add("argument of ntile must be a positive integer"); // fts5 functions errors.add("unable to use function highlight in the requested context"); @@ -62,9 +75,21 @@ public static void addExpectedExpressionErrors(Collection errors) { errors.add("ORDER BY clause should come after"); errors.add("LIMIT clause should come after"); + errors.add("unsafe use of load_extension"); + errors.add("table does not support scanning"); + errors.add("circularly defined"); + errors.add("[SQLITE_ERROR] SQL error or missing database"); // A possible delay in the execution of DROP TABLE + // statement. + return errors; + } + + public static void addExpectedExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExpressionErrors()); } - public static void addMatchQueryErrors(Collection errors) { + public static List getMatchQueryErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("unable to use function MATCH in the requested context"); errors.add("malformed MATCH expression"); errors.add("fts5: syntax error near"); @@ -73,25 +98,80 @@ public static void addMatchQueryErrors(Collection errors) { errors.add("fts5: column queries are not supported"); // vt0.c0 MATCH '2016456922' errors.add("fts5: phrase queries are not supported"); errors.add("unterminated string"); + + return errors; + } + + public static void addMatchQueryErrors(ExpectedErrors errors) { + errors.addAll(getMatchQueryErrors()); } - public static void addTableManipulationErrors(List errors) { + public static List getTableManipulationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("unsupported frame specification"); errors.add("non-deterministic functions prohibited in CHECK constraints"); errors.addAll(Arrays.asList("subqueries prohibited in CHECK constraints", "generated columns cannot be part of the PRIMARY KEY", "must have at least one non-generated column")); + + return errors; } - public static void addQueryErrors(Set errors) { + public static void addTableManipulationErrors(ExpectedErrors errors) { + errors.addAll(getTableManipulationErrors()); + } + + public static List getQueryErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("ON clause references tables to its right"); + + return errors; + } + + public static void addQueryErrors(ExpectedErrors errors) { + errors.addAll(getQueryErrors()); } - public static void addInsertNowErrors(List errors) { + public static List getInsertNowErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("non-deterministic use of strftime()"); errors.add("non-deterministic use of time()"); errors.add("non-deterministic use of datetime()"); errors.add("non-deterministic use of julianday()"); errors.add("non-deterministic use of date()"); + + return errors; + } + + public static void addInsertNowErrors(ExpectedErrors errors) { + errors.addAll(getInsertNowErrors()); + } + + public static List getInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("String or BLOB exceeds size limit"); + errors.add("[SQLITE_CONSTRAINT_CHECK]"); + errors.add("[SQLITE_CONSTRAINT_PRIMARYKEY]"); + errors.add("[SQLITE_CONSTRAINT]"); + errors.add("[SQLITE_CONSTRAINT_NOTNULL]"); + errors.add("[SQLITE_CONSTRAINT_UNIQUE]"); + errors.add("cannot INSERT into generated column"); // TODO: filter out generated columns + errors.add("A table in the database is locked"); // https://www.sqlite.org/src/tktview?name=56a74875be + errors.add("The database file is locked"); + errors.add("too many levels of trigger recursion"); + errors.add("cannot UPDATE generated column"); + errors.add("[SQLITE_ERROR] SQL error or missing database (no such table:"); + errors.add("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch"); + errors.add("no such column"); // trigger + + return errors; + } + + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getInsertUpdateErrors()); } } diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index fd090b60a..7ca82ec02 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -14,14 +14,21 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -47,9 +54,12 @@ private void print(SQLite3Expression expr) { sb.append("\t"); } sb.append(v.get()); - sb.append(" -- " + expr.getExpectedValue()); - sb.append(" explicit collate: " + expr.getExplicitCollateSequence()); - sb.append(" implicit collate: " + expr.getImplicitCollateSequence()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append(" explicit collate: "); + sb.append(expr.getExplicitCollateSequence()); + sb.append(" implicit collate: "); + sb.append(expr.getImplicitCollateSequence()); sb.append("\n"); } @@ -300,4 +310,43 @@ public void visit(SQLite3SetClause set) { visit(set.getRight()); } + @Override + public void visit(SQLite3Alias alias) { + print(alias); + print(alias.getOriginalExpression()); + print(alias.getAliasExpression()); + } + + @Override + public void visit(SQLite3WithClause withClause) { + print(withClause); + print(withClause.getLeft()); + print(withClause.getRight()); + } + + @Override + public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { + print(tableAndColumnRef); + } + + @Override + public void visit(SQLite3Values values) { + print(values); + } + + @Override + public void visit(SQLite3ExpressionBag expr) { + print(expr); + print(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3Typeof expr) { + print(expr); + print(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3ResultMap tableSummary) { + } } diff --git a/src/sqlancer/sqlite3/SQLite3GlobalState.java b/src/sqlancer/sqlite3/SQLite3GlobalState.java new file mode 100644 index 000000000..ea8bd5f77 --- /dev/null +++ b/src/sqlancer/sqlite3/SQLite3GlobalState.java @@ -0,0 +1,15 @@ +package sqlancer.sqlite3; + +import java.sql.SQLException; + +import sqlancer.SQLGlobalState; +import sqlancer.sqlite3.schema.SQLite3Schema; + +public class SQLite3GlobalState extends SQLGlobalState { + + @Override + protected SQLite3Schema readSchema() throws SQLException { + return SQLite3Schema.fromConnection(this); + } + +} diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index f570da048..e9e34892e 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -1,27 +1,15 @@ package sqlancer.sqlite3; -import java.sql.SQLException; -import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; -import sqlancer.MainOptions.DBMSConverter; -import sqlancer.TestOracle; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; -import sqlancer.sqlite3.oracle.SQLite3Fuzzer; -import sqlancer.sqlite3.oracle.SQLite3NoRECOracle; -import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPDistinctOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPGroupByOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPHavingOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPWhereOracle; +import sqlancer.DBMSSpecificOptions; @Parameters(separators = "=", commandDescription = "SQLite3") -public class SQLite3Options { +public class SQLite3Options implements DBMSSpecificOptions { @Parameter(names = { "--test-fts" }, description = "Test the FTS extensions", arity = 1) public boolean testFts = true; @@ -66,12 +54,15 @@ public class SQLite3Options { @Parameter(names = { "--test-match" }, description = "Allow the generation of the MATCH operator", arity = 1) public boolean testMatch = true; + @Parameter(names = { "--test-in-operator" }, description = "Allow the generation of the IN operator", arity = 1) + public boolean testIn = true; + @Parameter(names = { "--test-distinct-in-view" }, description = "DISTINCT in views might cause occasional false positives in NoREC and TLP", arity = 1) public boolean testDistinctInView; - @Parameter(names = "--oracle", converter = DBMSConverter.class) - public SQLite3Oracle oracle = SQLite3Oracle.NoREC; + @Parameter(names = "--oracle") + public SQLite3OracleFactory oracles = SQLite3OracleFactory.NoREC; @Parameter(names = { "--delete-existing-databases" }, description = "Delete a database file if it already exists", arity = 1) @@ -82,89 +73,34 @@ public class SQLite3Options { public boolean generateDatabase = true; @Parameter(names = { - "--print-statements" }, description = "Specifies whether to print SQL statements to stdout", arity = 1) - public boolean printStatements; + "--max-num-tables" }, description = "The maximum number of tables/virtual tables/ rtree tables/ views that can be created") + public int maxNumTables = 10; - @Parameter(names = { - "--execute-queries" }, description = "Specifies whether the query in the fuzzer should be executed", arity = 1) - public boolean executeQuery = true; + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; - @Parameter(names = { - "--print-successful-statements" }, description = "Specifies whether to print SQL statements to stdout", arity = 1) - public boolean executeStatementsAndPrintSuccessfulOnes; + public enum CODDTestModel { + RANDOM, EXPRESSION, SUBQUERY; - @Parameter(names = { - "--exit-after-first-database" }, description = "Specifies whether to stop SQLancer after creating and executing the test oracle on the first database", arity = 1) - public boolean exitAfterFirstDatabase; - - public enum SQLite3Oracle { - PQS { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3PivotedQuerySynthesisOracle(globalState); - } - }, - NoREC { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3NoRECOracle(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPAggregateOracle(globalState); - } - - }, - WHERE { - - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPWhereOracle(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPDistinctOracle(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPGroupByOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPHavingOracle(globalState); - } - }, - FUZZER { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3Fuzzer(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); - oracles.add(new SQLite3TLPWhereOracle(globalState)); - oracles.add(new SQLite3TLPDistinctOracle(globalState)); - oracles.add(new SQLite3TLPGroupByOracle(globalState)); - oracles.add(new SQLite3TLPHavingOracle(globalState)); - oracles.add(new SQLite3TLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles); - } - }; - - public abstract TestOracle create(SQLite3GlobalState globalState) throws SQLException; + public boolean isRandom() { + return this == RANDOM; + } + + public boolean isExpression() { + return this == EXPRESSION; + } + + public boolean isSubquery() { + return this == SUBQUERY; + } + } + + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on EXPRESSION, SUBQUERY, or RANDOM") + public CODDTestModel coddTestModel = CODDTestModel.RANDOM; + @Override + public List getTestOracleFactory() { + return Arrays.asList(oracles); } } diff --git a/src/sqlancer/sqlite3/SQLite3OracleFactory.java b/src/sqlancer/sqlite3/SQLite3OracleFactory.java new file mode 100644 index 000000000..2e2f9f0f7 --- /dev/null +++ b/src/sqlancer/sqlite3/SQLite3OracleFactory.java @@ -0,0 +1,113 @@ +package sqlancer.sqlite3; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.oracle.SQLite3CODDTestOracle; +import sqlancer.sqlite3.oracle.SQLite3Fuzzer; +import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPDistinctOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPGroupByOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPHavingOracle; + +public enum SQLite3OracleFactory implements OracleFactory { + PQS { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3PivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + + }, + NoREC { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .with(SQLite3Errors.getMatchQueryErrors()).with(SQLite3Errors.getQueryErrors()) + .with("misuse of aggregate", "misuse of window function", + "second argument to nth_value must be a positive integer", "no such table", + "no query solution", "unable to use function MATCH in the requested context") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + AGGREGATE { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPAggregateOracle(globalState); + } + + }, + WHERE { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .build(); + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + DISTINCT { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPDistinctOracle(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPGroupByOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPHavingOracle(globalState); + } + }, + FUZZER { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3Fuzzer(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CODDTest { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3CODDTestOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }; + +} diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 2249c0d1e..5fbd3b471 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -1,33 +1,29 @@ package sqlancer.sqlite3; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.sql.Connection; import java.sql.DriverManager; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; -import sqlancer.GlobalState; +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; -import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; import sqlancer.Randomly; -import sqlancer.StateToReproduce; -import sqlancer.StateToReproduce.SQLite3StateToReproduce; -import sqlancer.TestOracle; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; -import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.gen.SQLite3CreateVirtualRtreeTabelGenerator; import sqlancer.sqlite3.gen.SQLite3ExplainGenerator; import sqlancer.sqlite3.gen.SQLite3PragmaGenerator; @@ -45,22 +41,16 @@ import sqlancer.sqlite3.gen.ddl.SQLite3ViewGenerator; import sqlancer.sqlite3.gen.dml.SQLite3DeleteGenerator; import sqlancer.sqlite3.gen.dml.SQLite3InsertGenerator; +import sqlancer.sqlite3.gen.dml.SQLite3StatTableGenerator; import sqlancer.sqlite3.gen.dml.SQLite3UpdateGenerator; -import sqlancer.sqlite3.schema.SQLite3Schema; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; -public class SQLite3Provider extends ProviderAdapter { +@AutoService(DatabaseProvider.class) +public class SQLite3Provider extends SQLProviderAdapter { public static boolean allowFloatingPointFp = true; public static boolean mustKnowResult; - private SQLite3StateToReproduce state; - private String databaseName; - - private SQLite3GlobalState globalState; - // PRAGMAS to achieve good performance private static final List DEFAULT_PRAGMAS = Arrays.asList("PRAGMA cache_size = 50000;", "PRAGMA temp_store=MEMORY;", "PRAGMA synchronous=off;"); @@ -69,138 +59,130 @@ public SQLite3Provider() { super(SQLite3GlobalState.class, SQLite3Options.class); } - public enum Action { - PRAGMA(SQLite3PragmaGenerator::insertPragma), // - INDEX(SQLite3IndexGenerator::insertIndex), // - INSERT(SQLite3InsertGenerator::insertRow), // - VACUUM(SQLite3VacuumGenerator::executeVacuum), // - REINDEX(SQLite3ReindexGenerator::executeReindex), // - ANALYZE(SQLite3AnalyzeGenerator::generateAnalyze), // - DELETE(SQLite3DeleteGenerator::deleteContent), // - TRANSACTION_START(SQLite3TransactionGenerator::generateBeginTransaction), // - ALTER(SQLite3AlterTable::alterTable), // - DROP_INDEX(SQLite3DropIndexGenerator::dropIndex), // - UPDATE(SQLite3UpdateGenerator::updateRow), // - ROLLBACK_TRANSACTION(SQLite3TransactionGenerator::generateRollbackTransaction), // - COMMIT(SQLite3TransactionGenerator::generateCommit), // - DROP_TABLE(SQLite3DropTableGenerator::dropTable), // - DROP_VIEW(SQLite3ViewGenerator::dropView), // - EXPLAIN(SQLite3ExplainGenerator::explain), // + public enum Action implements AbstractAction { + PRAGMA(SQLite3PragmaGenerator::insertPragma), // 0 + CREATE_INDEX(SQLite3IndexGenerator::insertIndex), // 1 + CREATE_VIEW(SQLite3ViewGenerator::generate), // 2 + CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // 3 + CREATE_TABLE(SQLite3TableGenerator::createRandomTableStatement), // 4 + CREATE_VIRTUALTABLE(SQLite3CreateVirtualFTSTableGenerator::createRandomTableStatement), // 5 + CREATE_RTREETABLE(SQLite3CreateVirtualRtreeTabelGenerator::createRandomTableStatement), // 6 + INSERT(SQLite3InsertGenerator::insertRow), // 7 + DELETE(SQLite3DeleteGenerator::deleteContent), // 8 + ALTER(SQLite3AlterTable::alterTable), // 9 + UPDATE(SQLite3UpdateGenerator::updateRow), // 10 + DROP_INDEX(SQLite3DropIndexGenerator::dropIndex), // 11 + DROP_TABLE(SQLite3DropTableGenerator::dropTable), // 12 + DROP_VIEW(SQLite3ViewGenerator::dropView), // 13 + VACUUM(SQLite3VacuumGenerator::executeVacuum), // 14 + REINDEX(SQLite3ReindexGenerator::executeReindex), // 15 + ANALYZE(SQLite3AnalyzeGenerator::generateAnalyze), // 16 + EXPLAIN(SQLite3ExplainGenerator::explain), // 17 CHECK_RTREE_TABLE((g) -> { SQLite3Table table = g.getSchema().getRandomTableOrBailout(t -> t.getName().startsWith("r")); String format = String.format("SELECT rtreecheck('%s');", table.getName()); - return new QueryAdapter(format); - }), // - VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // - CREATE_VIEW(SQLite3ViewGenerator::generate), // - CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // - MANIPULATE_STAT_TABLE((g) -> { - List columns = new ArrayList<>(); - SQLite3Table t = new SQLite3Table("sqlite_stat1", columns, TableKind.MAIN, false, 1, false, false, false); - if (Randomly.getBoolean()) { - return SQLite3DeleteGenerator.deleteContent(g, t); - } else { - StringBuilder sb = new StringBuilder(); - sb.append("INSERT OR IGNORE INTO sqlite_stat1"); - String indexName; - try (Statement stat = g.getConnection().createStatement()) { - try (ResultSet rs = stat.executeQuery( - "SELECT name FROM sqlite_master WHERE type='index' ORDER BY RANDOM() LIMIT 1;")) { - if (rs.isClosed()) { - throw new IgnoreMeException(); - } - indexName = rs.getString("name"); - } - } - sb.append(" VALUES"); - sb.append("('"); - sb.append(g.getSchema().getRandomTable().getName()); - sb.append("', "); - sb.append("'"); - if (Randomly.getBoolean()) { - sb.append(indexName); - } else { - sb.append(g.getSchema().getRandomTable().getName()); - } - sb.append("'"); - sb.append(", '"); - for (int i = 0; i < Randomly.smallNumber(); i++) { - if (i != 0) { - sb.append(" "); - } - if (Randomly.getBoolean()) { - sb.append(g.getRandomly().getInteger()); - } else { - sb.append(Randomly.smallNumber()); - } - } - if (Randomly.getBoolean()) { - sb.append(" sz="); - sb.append(g.getRandomly().getInteger()); - } - if (Randomly.getBoolean()) { - sb.append(" unordered"); - } - if (Randomly.getBoolean()) { - sb.append(" noskipscan"); - } - sb.append("')"); - return new QueryAdapter(sb.toString(), Arrays.asList("no such table")); + return new SQLQueryAdapter(format, ExpectedErrors.from("The database file is locked")); + }), // 18 + VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // 19 + MANIPULATE_STAT_TABLE(SQLite3StatTableGenerator::getQuery), // 20 + TRANSACTION_START(SQLite3TransactionGenerator::generateBeginTransaction) { + @Override + public boolean canBeRetried() { + return false; } - }); - - private final QueryProvider queryProvider; - - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; - } - - public Query getQuery(SQLite3GlobalState state) throws SQLException { - return queryProvider.getQuery(state); - } - } - - public static class SQLite3GlobalState extends GlobalState { - - private SQLite3Schema schema; - private SQLite3Options sqliteOptions; - public SQLite3Schema getSchema() { - return schema; - } + }, // 21 + ROLLBACK_TRANSACTION(SQLite3TransactionGenerator::generateRollbackTransaction) { + @Override + public boolean canBeRetried() { + return false; + } + }, // 22 + COMMIT(SQLite3TransactionGenerator::generateCommit) { + @Override + public boolean canBeRetried() { + return false; + } + }; // 23 - public void setSchema(SQLite3Schema schema) { - this.schema = schema; - } + private final SQLQueryProvider sqlQueryProvider; - public void setSqliteOptions(SQLite3Options sqliteOptions) { - this.sqliteOptions = sqliteOptions; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } - public SQLite3Options getSqliteOptions() { - return sqliteOptions; + @Override + public SQLQueryAdapter getQuery(SQLite3GlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } - } private enum TableType { NORMAL, FTS, RTREE } + private static int mapActions(SQLite3GlobalState globalState, Action a) { + int nrPerformed = 0; + Randomly r = globalState.getRandomly(); + switch (a) { + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case DELETE: + case DROP_VIEW: + case DROP_INDEX: + nrPerformed = r.getInteger(0, 0); + break; + case ALTER: + nrPerformed = r.getInteger(0, 0); + break; + case EXPLAIN: + case CREATE_TRIGGER: + case DROP_TABLE: + nrPerformed = r.getInteger(0, 0); + break; + case VACUUM: + case CHECK_RTREE_TABLE: + nrPerformed = r.getInteger(0, 3); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + case MANIPULATE_STAT_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case VIRTUAL_TABLE_ACTION: + case UPDATE: + nrPerformed = r.getInteger(0, 30); + break; + case PRAGMA: + nrPerformed = r.getInteger(0, 20); + break; + case CREATE_TABLE: + case CREATE_VIRTUALTABLE: + case CREATE_RTREETABLE: + nrPerformed = 0; + break; + case TRANSACTION_START: + case REINDEX: + case ANALYZE: + case ROLLBACK_TRANSACTION: + case COMMIT: + default: + nrPerformed = r.getInteger(1, 10); + break; + } + return nrPerformed; + } + @Override - public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLException { - this.globalState = globalState; - SQLite3Options sqliteOptions = globalState.getDmbsSpecificOptions(); - QueryManager manager = globalState.getManager(); - MainOptions options = globalState.getOptions(); - this.databaseName = globalState.getDatabaseName(); + public void generateDatabase(SQLite3GlobalState globalState) throws Exception { Randomly r = new Randomly(SQLite3SpecialStringGenerator::generate); - globalState.setSqliteOptions(sqliteOptions); globalState.setRandomly(r); - StateLogger logger = globalState.getLogger(); - this.state = (SQLite3StateToReproduce) globalState.getState(); - globalState.setState(state); - if (globalState.getDmbsSpecificOptions().generateDatabase) { + if (globalState.getDbmsSpecificOptions().generateDatabase) { addSensiblePragmaDefaults(globalState); int nrTablesToCreate = 1; @@ -212,199 +194,65 @@ public void generateAndTestDatabase(SQLite3GlobalState globalState) throws SQLEx } int i = 0; - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); do { - Query tableQuery = getTableQuery(r, i++); - executeStatement(globalState, manager, tableQuery); - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); - } while (globalState.getSchema().getDatabaseTables().size() != nrTablesToCreate); + SQLQueryAdapter tableQuery = getTableQuery(globalState, i++); + globalState.executeStatement(tableQuery); + } while (globalState.getSchema().getDatabaseTables().size() < nrTablesToCreate); assert globalState.getSchema().getTables().getTables().size() == nrTablesToCreate; checkTablesForGeneratedColumnLoops(globalState); - if (globalState.getDmbsSpecificOptions().testDBStats && Randomly.getBooleanWithSmallProbability()) { - QueryAdapter tableQuery = new QueryAdapter( + if (globalState.getDbmsSpecificOptions().testDBStats && Randomly.getBooleanWithSmallProbability()) { + SQLQueryAdapter tableQuery = new SQLQueryAdapter( "CREATE VIRTUAL TABLE IF NOT EXISTS stat USING dbstat(main)"); - executeStatement(globalState, manager, tableQuery); - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); - } - int[] nrRemaining = new int[Action.values().length]; - List actions = new ArrayList<>(); - int total = 0; - for (i = 0; i < Action.values().length; i++) { - Action action = Action.values()[i]; - int nrPerformed = 0; - switch (action) { - case CREATE_VIEW: - nrPerformed = r.getInteger(0, 2); - break; - case DELETE: - case DROP_VIEW: - case DROP_INDEX: - nrPerformed = r.getInteger(0, 0); - break; - case ALTER: - nrPerformed = r.getInteger(0, 0); - break; - case EXPLAIN: - case CREATE_TRIGGER: - case DROP_TABLE: - nrPerformed = r.getInteger(0, 0); - break; - case VACUUM: - case CHECK_RTREE_TABLE: - nrPerformed = r.getInteger(0, 3); - break; - case INSERT: - nrPerformed = r.getInteger(0, options.getMaxNumberInserts()); - break; - case MANIPULATE_STAT_TABLE: - nrPerformed = r.getInteger(0, 5); - break; - case INDEX: - nrPerformed = r.getInteger(0, 5); - break; - case VIRTUAL_TABLE_ACTION: - case UPDATE: - nrPerformed = r.getInteger(0, 30); - break; - case PRAGMA: - nrPerformed = r.getInteger(0, 20); - break; - case TRANSACTION_START: - case REINDEX: - case ANALYZE: - case ROLLBACK_TRANSACTION: - case COMMIT: - default: - nrPerformed = r.getInteger(1, 10); - break; - } - if (nrPerformed != 0) { - actions.add(action); - } - nrRemaining[action.ordinal()] = nrPerformed; - total += nrPerformed; - } - - if (options.logEachSelect()) { - logger.writeCurrent(state); + globalState.executeStatement(tableQuery); } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + SQLite3Provider::mapActions, (q) -> { + if (q.couldAffectSchema() && globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); - while (total != 0) { - Action nextAction = null; - int selection = r.getInteger(0, total); - int previousRange = 0; - for (i = 0; i < nrRemaining.length; i++) { - if (previousRange <= selection && selection < previousRange + nrRemaining[i]) { - nextAction = Action.values()[i]; - break; - } else { - previousRange += nrRemaining[i]; - } - } - assert nextAction != null; - assert nrRemaining[nextAction.ordinal()] > 0; - nrRemaining[nextAction.ordinal()]--; - Query query = null; - try { - query = nextAction.getQuery(globalState); - if (options.logEachSelect()) { - logger.writeCurrent(query.getQueryString()); - } - executeStatement(globalState, manager, query); - } catch (IgnoreMeException e) { - - } - if (query != null && query.couldAffectSchema()) { - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - } - total--; - } - Query query = SQLite3TransactionGenerator.generateCommit(globalState); - executeStatement(globalState, manager, query); + SQLQueryAdapter query = SQLite3TransactionGenerator.generateCommit(globalState); + globalState.executeStatement(query); // also do an abort for DEFERRABLE INITIALLY DEFERRED query = SQLite3TransactionGenerator.generateRollbackTransaction(globalState); - executeStatement(globalState, manager, query); - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); - manager.incrementCreateDatabase(); - } else { - globalState.setSchema(SQLite3Schema.fromConnection(globalState)); - } - TestOracle oracle = globalState.getSqliteOptions().oracle.create(globalState); - if (oracle.onlyWorksForNonEmptyTables()) { - for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { - int nrRows = SQLite3Schema.getNrRows(globalState, table.getName()); - if (nrRows == 0) { - throw new IgnoreMeException(); - } - } - } - for (int i = 0; i < options.getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - - } - } - try { - if (options.logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; - } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - if (globalState.getDmbsSpecificOptions().exitAfterFirstDatabase) { - System.exit(0); - } - } - - private void executeStatement(SQLite3GlobalState globalState, QueryManager manager, Query tableQuery) - throws SQLException { - manager.execute(tableQuery); - if (globalState.getDmbsSpecificOptions().printStatements) { - String s = tableQuery.getQueryString(); - if (!s.endsWith(";")) { - s = s + ";"; - } - System.out.println(s); + globalState.executeStatement(query); } } - private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws SQLException { + private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) throws Exception { for (SQLite3Table table : globalState.getSchema().getDatabaseTables()) { - Query q = new QueryAdapter("SELECT * FROM " + table.getName(), - Arrays.asList("needs an odd number of arguments", " requires an even number of arguments", + SQLQueryAdapter q = new SQLQueryAdapter("SELECT * FROM " + table.getName(), + ExpectedErrors.from("needs an odd number of arguments", " requires an even number of arguments", "generated column loop", "integer overflow", "malformed JSON", - "JSON cannot hold BLOB values", "JSON path error", "labels must be TEXT")); + "JSON cannot hold BLOB values", "JSON path error", "labels must be TEXT", + "table does not support scanning")); if (!q.execute(globalState)) { throw new IgnoreMeException(); } } } - private Query getTableQuery(Randomly r, int i) throws AssertionError { - Query tableQuery; + private SQLQueryAdapter getTableQuery(SQLite3GlobalState globalState, int i) throws AssertionError { + SQLQueryAdapter tableQuery; List options = new ArrayList<>(Arrays.asList(TableType.values())); - if (!globalState.getSqliteOptions().testFts) { + if (!globalState.getDbmsSpecificOptions().testFts) { options.remove(TableType.FTS); } - if (!globalState.getSqliteOptions().testRtree) { + if (!globalState.getDbmsSpecificOptions().testRtree) { options.remove(TableType.RTREE); } switch (Randomly.fromList(options)) { case NORMAL: - String tableName = SQLite3Common.createTableName(i); + String tableName = DBMSCommon.createTableName(i); tableQuery = SQLite3TableGenerator.createTableStatement(tableName, globalState); break; case FTS: - String ftsTableName = "v" + SQLite3Common.createTableName(i); - tableQuery = SQLite3CreateVirtualFTSTableGenerator.createTableStatement(ftsTableName, r); + String ftsTableName = "v" + DBMSCommon.createTableName(i); + tableQuery = SQLite3CreateVirtualFTSTableGenerator.createTableStatement(ftsTableName, + globalState.getRandomly()); break; case RTREE: String rTreeTableName = "rt" + i; @@ -416,40 +264,37 @@ private Query getTableQuery(Randomly r, int i) throws AssertionError { return tableQuery; } - private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws SQLException { + private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws Exception { List pragmasToExecute = new ArrayList<>(); if (!Randomly.getBooleanWithSmallProbability()) { pragmasToExecute.addAll(DEFAULT_PRAGMAS); } - if (Randomly.getBoolean() && !mustKnowResult) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { + // the PQS implementation currently assumes the default behavior of LIKE pragmasToExecute.add("PRAGMA case_sensitive_like=ON;"); } - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { + // the encoding has an influence how binary strings are cast pragmasToExecute.add(String.format("PRAGMA encoding = '%s';", Randomly.fromOptions("UTF-8", "UTF-16", "UTF-16le", "UTF-16be"))); } for (String s : pragmasToExecute) { - if (globalState.getDmbsSpecificOptions().printStatements) { - System.out.println(s); - } - Query q = new QueryAdapter(s); - state.statements.add(q); - q.execute(globalState); + globalState.executeStatement(new SQLQueryAdapter(s)); } } @Override - public Connection createDatabase(SQLite3GlobalState globalState) throws SQLException { + public SQLConnection createDatabase(SQLite3GlobalState globalState) throws SQLException { File dir = new File("." + File.separator + "databases"); if (!dir.exists()) { dir.mkdir(); } File dataBase = new File(dir, globalState.getDatabaseName() + ".db"); - if (dataBase.exists() && ((SQLite3GlobalState) globalState).getDmbsSpecificOptions().deleteIfExists) { + if (dataBase.exists() && ((SQLite3GlobalState) globalState).getDbmsSpecificOptions().deleteIfExists) { dataBase.delete(); } String url = "jdbc:sqlite:" + dataBase.getAbsolutePath(); - return DriverManager.getConnection(url); + return new SQLConnection(DriverManager.getConnection(url)); } @Override @@ -458,17 +303,57 @@ public String getDBMSName() { } @Override - public String toString() { - return String.format("SQLite3Provider [database: %s]", databaseName); + public String getQueryPlan(String selectStr, SQLite3GlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + // Set up the expected errors for NoREC oracle. + ExpectedErrors errors = new ExpectedErrors(); + SQLite3Errors.addExpectedExpressionErrors(errors); + SQLite3Errors.addMatchQueryErrors(errors); + SQLite3Errors.addQueryErrors(errors); + SQLite3Errors.addInsertUpdateErrors(errors); + + SQLQueryAdapter q = new SQLQueryAdapter(SQLite3ExplainGenerator.explain(selectStr), errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + queryPlan += rs.getString(4) + ";"; + } + } + } catch (SQLException | AssertionError e) { + queryPlan = ""; + } + return queryPlan; } @Override - public void printDatabaseSpecificState(FileWriter writer, StateToReproduce state) { + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; } @Override - public StateToReproduce getStateToReproduce(String databaseName) { - return new SQLite3StateToReproduce(databaseName); + protected void executeMutator(int index, SQLite3GlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + @Override + protected boolean addRowsToAllTables(SQLite3GlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (SQLite3Table table : tablesNoRow) { + SQLQueryAdapter queryAddRows = SQLite3InsertGenerator.insertRow(globalState, table); + globalState.executeStatement(queryAddRows); + } + + return true; + } } diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 2b3368634..8fdbf4438 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -1,14 +1,20 @@ package sqlancer.sqlite3; import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; import sqlancer.sqlite3.ast.SQLite3Case.CasePair; import sqlancer.sqlite3.ast.SQLite3Case.SQLite3CaseWithBaseExpression; import sqlancer.sqlite3.ast.SQLite3Case.SQLite3CaseWithoutBaseExpression; +import sqlancer.sqlite3.ast.SQLite3Cast; import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.ast.SQLite3Constant.SQLite3NullConstant; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.BetweenOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Cast; @@ -17,12 +23,19 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; @@ -33,7 +46,7 @@ import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecBetween; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecTerm; -import sqlancer.visitor.ToStringVisitor; +import sqlancer.sqlite3.schema.SQLite3DataType; public class SQLite3ToStringVisitor extends ToStringVisitor implements SQLite3Visitor { @@ -98,6 +111,10 @@ public void visit(SQLite3Select s, boolean inner) { if (inner) { sb.append("("); } + if (s.getWithClause() != null) { + visit(s.getWithClause()); + sb.append(" "); + } sb.append("SELECT "); switch (s.getFromOptions()) { case DISTINCT: @@ -138,7 +155,7 @@ public void visit(SQLite3Select s, boolean inner) { visit(whereClause); sb.append(")"); } - if (s.getGroupByClause().size() > 0) { + if (!s.getGroupByClause().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); visit(s.getGroupByClause()); @@ -147,9 +164,9 @@ public void visit(SQLite3Select s, boolean inner) { sb.append(" HAVING "); visit(s.getHavingClause()); } - if (!s.getOrderByClause().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByClause()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); @@ -176,7 +193,7 @@ public void visit(SQLite3Constant c) { // sb.append(c.asInt() == 1 ? "TRUE" : "FALSE"); // } else { // - 0X8000000000000000 results in an error message otherwise - if (Randomly.getBoolean() || c.asInt() == Long.MIN_VALUE) { + if (!c.isHex() || c.asInt() == Long.MIN_VALUE) { sb.append(c.asInt()); } else { long intVal = c.asInt(); @@ -209,7 +226,7 @@ public void visit(SQLite3Constant c) { if (c.getValue() instanceof byte[]) { arr = c.asBinary(); } else { - arr = c.asString().getBytes(); + arr = c.asString().getBytes(SQLite3Cast.DEFAULT_ENCODING); } sb.append(SQLite3Visitor.byteArrayToHex(arr)); sb.append("'"); @@ -236,6 +253,12 @@ public void visit(Join join) { case OUTER: sb.append("LEFT OUTER"); break; + case RIGHT: + sb.append("RIGHT OUTER"); + break; + case FULL: + sb.append("FULL OUTER"); + break; default: throw new AssertionError(join.getType()); } @@ -281,13 +304,20 @@ public void visit(InOperation op) { sb.append("("); visit(op.getLeft()); sb.append(" IN "); - sb.append("("); if (op.getRightExpressionList() != null) { + sb.append("("); visit(op.getRightExpressionList()); + sb.append(")"); } else { - visit(op.getRightSelect()); + if (op.getRightSelect() instanceof SQLite3Expression.SQLite3TableReference) { + visit(op.getRightSelect()); + } else { + sb.append("("); + visit(op.getRightSelect()); + sb.append(")"); + } } - sb.append(")"); + sb.append(")"); } @@ -298,6 +328,9 @@ public void visit(Subquery query) { @Override public void visit(SQLite3Exist exist) { + if (exist.getNegated()) { + sb.append(" NOT"); + } sb.append(" EXISTS "); if (exist.getExpression() instanceof SQLite3SetClause) { sb.append("("); @@ -475,4 +508,104 @@ public void visit(SQLite3SetClause set) { sb.append(SQLite3Visitor.asString(set.getRight())); } + @Override + public void visit(SQLite3Alias alias) { + sb.append("("); + visit(alias.getOriginalExpression()); + sb.append(")"); + sb.append(" AS "); + visit(alias.getAliasExpression()); + } + + @Override + public void visit(SQLite3WithClause withClause) { + sb.append("WITH "); + visit(withClause.getLeft()); + sb.append(" AS "); + visit(withClause.getRight()); + } + + @Override + public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { + sb.append(tableAndColumnRef.getString()); + } + + @Override + public void visit(SQLite3Values values) { + Map> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + List columnNames = values.getColumns().stream().map(c -> c.getName()).collect(Collectors.toList()); + sb.append("(VALUES "); + for (int i = 0; i < size; i++) { + sb.append("("); + Boolean isFirstColumn = true; + for (String name : columnNames) { + if (!isFirstColumn) { + sb.append(", "); + } + if (vs.get(name).get(i).getDataType() == SQLite3DataType.NULL) { + visit(vs.get(name).get(i)); + } else { + sb.append("(CAST("); + visit(vs.get(name).get(i)); + sb.append(" AS "); + sb.append(vs.get(name).get(i).getDataType().toString()); + sb.append("))"); + } + isFirstColumn = false; + } + sb.append(")"); + if (i < size - 1) { + sb.append(", "); + } + } + sb.append(")"); + } + + @Override + public void visit(SQLite3ExpressionBag expr) { + visit(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3Typeof expr) { + sb.append("typeof("); + visit(expr.getInnerExpr()); + sb.append(")"); + } + + @Override + public void visit(SQLite3ResultMap tableSummary) { + // We use the CASE WHEN THEN END expression to represent the result of an expression for each row in the table. + SQLite3Values values = tableSummary.getValues(); + List columnRefs = tableSummary.getColumns(); + List summary = tableSummary.getSummary(); + + Map> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + if (size == 0) { + throw new AssertionError("The result of the expression must not be empty."); + } + List columnNames = values.getColumns().stream().map(c -> c.getName()).collect(Collectors.toList()); + sb.append(" CASE "); + for (int i = 0; i < size; i++) { + sb.append("WHEN "); + for (int j = 0; j < columnNames.size(); ++j) { + visit(columnRefs.get(j)); + if (vs.get(columnNames.get(j)).get(i) instanceof SQLite3NullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(vs.get(columnNames.get(j)).get(i).toString()); + } + if (j < columnNames.size() - 1) { + sb.append(" AND "); + } + } + sb.append(" THEN "); + visit(summary.get(i)); + sb.append(" "); + } + sb.append("END "); + } } diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index f891c17f0..02ac2b4d7 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -13,14 +13,21 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -130,6 +137,20 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3WindowFunctionFrameSpecBetween between); + void visit(SQLite3Alias alias); + + void visit(SQLite3WithClause withClause); + + void visit(SQLite3TableAndColumnRef tableAndColumnRef); + + void visit(SQLite3Values values); + + void visit(SQLite3ExpressionBag expr); + + void visit(SQLite3Typeof expr); + + void visit(SQLite3ResultMap tableSummary); + default void visit(SQLite3Expression expr) { if (expr instanceof Sqlite3BinaryOperation) { visit((Sqlite3BinaryOperation) expr); @@ -193,6 +214,20 @@ default void visit(SQLite3Expression expr) { visit((SQLite3TableReference) expr); } else if (expr instanceof SQLite3SetClause) { visit((SQLite3SetClause) expr); + } else if (expr instanceof SQLite3Alias) { + visit((SQLite3Alias) expr); + } else if (expr instanceof SQLite3WithClause) { + visit((SQLite3WithClause) expr); + } else if (expr instanceof SQLite3TableAndColumnRef) { + visit((SQLite3TableAndColumnRef) expr); + } else if (expr instanceof SQLite3Values) { + visit((SQLite3Values) expr); + } else if (expr instanceof SQLite3ExpressionBag) { + visit((SQLite3ExpressionBag) expr); + } else if (expr instanceof SQLite3Typeof) { + visit((SQLite3Typeof) expr); + } else if (expr instanceof SQLite3ResultMap) { + visit((SQLite3ResultMap) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Aggregate.java b/src/sqlancer/sqlite3/ast/SQLite3Aggregate.java index d1e036b52..a9129d489 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Aggregate.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Aggregate.java @@ -11,7 +11,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; /** - * @see https://www.sqlite.org/lang_aggfunc.html + * @see Built-in Aggregate Functions */ public class SQLite3Aggregate extends SQLite3Expression { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Case.java b/src/sqlancer/sqlite3/ast/SQLite3Case.java index 4f5aef30c..da73cfe18 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Case.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Case.java @@ -120,15 +120,15 @@ public SQLite3Constant getExpectedValue() { SQLite3Constant whenComparisonValue = c.getCond().getExpectedValue(); if (whenComparisonValue == null) { return null; - } // TODO collate + } SQLite3CollateSequence seq; if (baseExpr.getExplicitCollateSequence() != null) { seq = baseExpr.getExplicitCollateSequence(); - } else if (whenComparisonValue.getExplicitCollateSequence() != null) { - seq = whenComparisonValue.getExplicitCollateSequence(); + } else if (c.getCond().getExplicitCollateSequence() != null) { + seq = c.getCond().getExplicitCollateSequence(); } else if (c.getCond().getImplicitCollateSequence() != null) { seq = c.getCond().getImplicitCollateSequence(); - } else if (whenComparisonValue.getImplicitCollateSequence() != null) { + } else if (c.getCond().getImplicitCollateSequence() != null) { seq = c.getCond().getImplicitCollateSequence(); } else { seq = SQLite3CollateSequence.BINARY; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Cast.java b/src/sqlancer/sqlite3/ast/SQLite3Cast.java index 5fb92d65c..5a33364a1 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Cast.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Cast.java @@ -1,17 +1,19 @@ package sqlancer.sqlite3.ast; import java.math.BigDecimal; +import java.nio.charset.Charset; import java.sql.Connection; -import java.sql.Statement; import java.util.Optional; import java.util.regex.Pattern; +import sqlancer.IgnoreMeException; import sqlancer.sqlite3.schema.SQLite3DataType; public final class SQLite3Cast { private static final double MAX_INT_FOR_WHICH_CONVERSION_TO_INT_IS_TRIED = Math.pow(2, 51 - 1) - 1; private static final double MIN_INT_FOR_WHICH_CONVERSION_TO_INT_IS_TRIED = -Math.pow(2, 51 - 1); + public static final Charset DEFAULT_ENCODING = Charset.forName("UTF-8"); private static final byte FILE_SEPARATOR = 0x1c; private static final byte GROUP_SEPARATOR = 0x1d; @@ -46,11 +48,18 @@ public static Optional isTrue(SQLite3Constant value) { } } + public static void checkDoubleIsInsideDangerousRange(double doubleVal) { + // high double-values might result in small rounding differences between Java and SQLite + if (Math.abs(doubleVal) > 1e15) { + throw new IgnoreMeException(); + } + } + // SELECT CAST('-1.370998801E9' AS INTEGER) == -1 public static SQLite3Constant castToInt(SQLite3Constant originalCons) { SQLite3Constant cons = originalCons; if (cons.getDataType() == SQLite3DataType.BINARY) { - String text = new String(cons.asBinary()); + String text = new String(cons.asBinary(), DEFAULT_ENCODING); cons = SQLite3Constant.createTextConstant(text); } switch (cons.getDataType()) { @@ -59,6 +68,7 @@ public static SQLite3Constant castToInt(SQLite3Constant originalCons) { case INT: return cons; case REAL: + checkDoubleIsInsideDangerousRange(cons.asDouble()); return SQLite3Constant.createIntConstant((long) cons.asDouble()); case TEXT: String asString = cons.asString(); @@ -100,7 +110,9 @@ public static SQLite3Constant castToInt(SQLite3Constant originalCons) { public static SQLite3Constant castToReal(SQLite3Constant cons) { SQLite3Constant numericValue = castToNumeric(cons); if (numericValue.getDataType() == SQLite3DataType.INT) { - return SQLite3Constant.createRealConstant(numericValue.asInt()); + double val = numericValue.asInt(); + checkDoubleIsInsideDangerousRange(val); + return SQLite3Constant.createRealConstant(val); } else { return numericValue; } @@ -114,7 +126,7 @@ public static SQLite3Constant castToNumericFromNumOperand(SQLite3Constant value) return convertInternal(value, false, false, false); } - /** + /* * Applies numeric affinity to a value. */ public static SQLite3Constant castToNumeric(SQLite3Constant value) { @@ -125,7 +137,7 @@ private static SQLite3Constant convertInternal(SQLite3Constant originalValue, bo boolean noNumIsRealZero, boolean convertIntToReal) throws AssertionError { SQLite3Constant value = originalValue; if (value.getDataType() == SQLite3DataType.BINARY) { - String text = new String(value.asBinary()); + String text = new String(value.asBinary(), DEFAULT_ENCODING); value = SQLite3Constant.createTextConstant(text); } switch (value.getDataType()) { @@ -149,7 +161,7 @@ private static SQLite3Constant convertInternal(SQLite3Constant originalValue, bo for (int i = asString.length(); i >= 0; i--) { try { String substring = asString.substring(0, i); - double d = Double.valueOf(substring); + double d = Double.parseDouble(substring); BigDecimal first = new BigDecimal(substring); long longValue = first.longValue(); BigDecimal second = BigDecimal.valueOf(longValue); @@ -235,54 +247,14 @@ public static SQLite3Constant castToText(SQLite3Constant cons) { return SQLite3Constant.createTextConstant("Inf"); } else if (cons.asDouble() == Double.NEGATIVE_INFINITY) { return SQLite3Constant.createTextConstant("-Inf"); + } else { + return null; } - return castRealToText(cons); - // if (true) { - // throw new IgnoreMeException(); - // } - // NumberFormat fmt = NumberFormat.getInstance(); - // fmt.setGroupingUsed(false); - // fmt.setMaximumIntegerDigits(10); - // fmt.setMinimumFractionDigits(1); - // fmt.setRoundingMode(RoundingMode.UNNECESSARY); - // int digits; - // if (cons.asDouble() < 0) { - // digits = 15; - // } else { - // digits = 15; - // } - // fmt.setMaximumFractionDigits(digits); - // try { - // String s = fmt.format(cons.asDouble()); - // if (s.contentEquals("")) { - // throw new IgnoreMeException(); - // } - // return SQLite3Constant.createTextConstant(s); - // } catch (Exception e) { - // throw new IgnoreMeException(); - // } } if (cons.getDataType() == SQLite3DataType.INT) { return SQLite3Constant.createTextConstant(String.valueOf(cons.asInt())); } - // if (cons.getDataType() == SQLite3DataType.BINARY) { - // try { - // return SQLite3Constant.createTextConstant(new String(cons.asBinary(), "UTF-8").replaceAll("\\p{C}", "")); - // } catch (UnsupportedEncodingException e) { - // throw new AssertionError(e); - // } - // } return null; - // throw new AssertionError(); - } - - private static synchronized SQLite3Constant castRealToText(SQLite3Constant cons) throws AssertionError { - try (Statement s = castDatabase.createStatement()) { - String castResult = s.executeQuery("SELECT CAST(" + cons.asDouble() + " AS TEXT)").getString(1); - return SQLite3Constant.createTextConstant(castResult); - } catch (Exception e) { - throw new AssertionError(e); - } } public static SQLite3Constant asBoolean(SQLite3Constant val) { @@ -302,7 +274,7 @@ public static SQLite3Constant castToBlob(SQLite3Constant cons) { if (stringVal == null) { return null; } else { - return SQLite3Constant.createBinaryConstant(stringVal.asString().getBytes()); + return SQLite3Constant.createBinaryConstant(stringVal.asString().getBytes(DEFAULT_ENCODING)); } } } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Constant.java b/src/sqlancer/sqlite3/ast/SQLite3Constant.java index abe3cd033..a6edd7bf9 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Constant.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Constant.java @@ -63,9 +63,21 @@ public SQLite3Constant applyLess(SQLite3Constant right, SQLite3CollateSequence c public static class SQLite3IntConstant extends SQLite3Constant { private final long value; + private final boolean isHex; + + public SQLite3IntConstant(long value, boolean isHex) { + this.value = value; + this.isHex = isHex; + } public SQLite3IntConstant(long value) { this.value = value; + this.isHex = false; + } + + @Override + public boolean isHex() { + return isHex; } @Override @@ -511,6 +523,10 @@ public SQLite3Constant applyLess(SQLite3Constant right, SQLite3CollateSequence c public abstract Object getValue(); + public boolean isHex() { + throw new UnsupportedOperationException(this.getDataType().toString()); + } + public long asInt() { throw new UnsupportedOperationException(this.getDataType().toString()); } @@ -533,6 +549,10 @@ public static SQLite3Constant createIntConstant(long val) { return new SQLite3IntConstant(val); } + public static SQLite3Constant createIntConstant(long val, boolean isHex) { + return new SQLite3IntConstant(val, isHex); + } + public static SQLite3Constant createBinaryConstant(byte[] val) { return new SQLite3BinaryConstant(val); } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index fa53cdeed..ad0f64ad6 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1,11 +1,15 @@ package sqlancer.sqlite3.ast; import java.util.List; +import java.util.Map; import java.util.Optional; import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.UnaryOperation; import sqlancer.sqlite3.SQLite3CollateHelper; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.ast.SQLite3Expression.BinaryComparisonOperation.BinaryComparisonOperator; @@ -15,10 +19,8 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -import sqlancer.visitor.BinaryOperation; -import sqlancer.visitor.UnaryOperation; -public abstract class SQLite3Expression { +public abstract class SQLite3Expression implements Expression { public static class SQLite3TableReference extends SQLite3Expression { @@ -92,17 +94,16 @@ public boolean isNumeric() { } } - /** + /* * See https://www.sqlite.org/datatype3.html 3.2 */ public TypeAffinity getAffinity() { return TypeAffinity.NONE; } - /** + /* * See https://www.sqlite.org/datatype3.html#assigning_collating_sequences_from_sql 7.1 * - * @return */ public abstract SQLite3CollateSequence getExplicitCollateSequence(); @@ -113,9 +114,19 @@ public SQLite3CollateSequence getImplicitCollateSequence() { public static class SQLite3Exist extends SQLite3Expression { private final SQLite3Expression select; + private boolean negated; - public SQLite3Exist(SQLite3Expression select) { + public SQLite3Exist(SQLite3Expression select, boolean negated) { this.select = select; + this.negated = negated; + } + + public void setNegated(boolean negated) { + this.negated = negated; + } + + public boolean getNegated() { + return this.negated; } public SQLite3Expression getExpression() { @@ -129,15 +140,22 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } - public static class Join extends SQLite3Expression { + public static class Join extends SQLite3Expression + implements sqlancer.common.ast.newast.Join { public enum JoinType { - INNER, CROSS, OUTER, NATURAL; + INNER, CROSS, OUTER, NATURAL, RIGHT, FULL; } private final SQLite3Table table; private SQLite3Expression onClause; - private final JoinType type; + private JoinType type; + + public Join(Join other) { + this.table = other.table; + this.onClause = other.onClause; + this.type = other.type; + } public Join(SQLite3Table table, SQLite3Expression onClause, JoinType type) { this.table = table; @@ -171,10 +189,14 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } + @Override public void setOnClause(SQLite3Expression onClause) { this.onClause = onClause; } + public void setType(JoinType type) { + this.type = type; + } } public static class Subquery extends SQLite3Expression { @@ -1126,50 +1148,7 @@ public SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { MULTIPLY("*") { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant result = compute(left, right); - if (result.getDataType() == SQLite3DataType.REAL) { - double val = result.asDouble(); - if (Double.isNaN(val)) { - return SQLite3Constant.createNullConstant(); - } - } - return result; - } - - private SQLite3Constant compute(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - try { - long intResult = Math.multiplyExact(leftInt, rightInt); - return SQLite3Constant.createIntConstant(intResult); - } catch (ArithmeticException e) { - double realResult = (double) leftInt * (double) rightInt; - return SQLite3Constant.createRealConstant(realResult); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftInt * rightDouble); - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - return SQLite3Constant.createRealConstant(leftReal * rightInt); - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftReal * rightReal); - } - } + return null; } }, @@ -1177,100 +1156,14 @@ private SQLite3Constant compute(SQLite3Constant left, SQLite3Constant right) { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - if (rightInt == 0) { - return SQLite3Constant.createNullConstant(); - } else if (leftInt == Long.MIN_VALUE && rightInt == -1) { - return SQLite3Constant.createRealConstant(9.22337203685478e+18); - } else { - return SQLite3Constant.createIntConstant(leftInt / rightInt); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - if (rightDouble == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant(leftInt / rightDouble); - } - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - if (rightInt == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant(leftReal / rightInt); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - if (rightReal == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant(leftReal / rightReal); - } - } - } + return null; } }, // division by zero results in zero REMAINDER("%") { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - if (rightInt == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createIntConstant(leftInt % rightInt); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - if ((long) rightDouble == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant(leftInt % (long) rightDouble); - } - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - if (rightInt == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant((long) leftReal % rightInt); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - if ((long) rightReal == 0) { - return SQLite3Constant.createNullConstant(); - } else { - return SQLite3Constant.createRealConstant((long) leftReal % (long) rightReal); - } - } - } + return null; } }, @@ -1279,80 +1172,15 @@ SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - try { - long intResult = Math.addExact(leftInt, rightInt); - return SQLite3Constant.createIntConstant(intResult); - } catch (ArithmeticException e) { - double realResult = (double) leftInt + (double) rightInt; - return SQLite3Constant.createRealConstant(realResult); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftInt + rightDouble); - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - return SQLite3Constant.createRealConstant(leftReal + rightInt); - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftReal + rightReal); - } - } + return null; } - }, MINUS("-") { @Override SQLite3Constant apply(SQLite3Constant left, SQLite3Constant right) { - SQLite3Constant leftNumeric = SQLite3Cast.castToNumericFromNumOperand(left); - SQLite3Constant rightNumeric = SQLite3Cast.castToNumericFromNumOperand(right); - if (leftNumeric.isNull() || rightNumeric.isNull()) { - return SQLite3Constant.createNullConstant(); - } - if (leftNumeric.getDataType() == SQLite3DataType.INT) { - long leftInt = leftNumeric.asInt(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - try { - long intResult = Math.subtractExact(leftInt, rightInt); - return SQLite3Constant.createIntConstant(intResult); - } catch (ArithmeticException e) { - double realResult = (double) leftInt - (double) rightInt; - return SQLite3Constant.createRealConstant(realResult); - } - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightDouble = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftInt - rightDouble); - } - } else { - assert leftNumeric.getDataType() == SQLite3DataType.REAL; - double leftReal = leftNumeric.asDouble(); - if (rightNumeric.getDataType() == SQLite3DataType.INT) { - long rightInt = rightNumeric.asInt(); - return SQLite3Constant.createRealConstant(leftReal - rightInt); - } else { - assert rightNumeric.getDataType() == SQLite3DataType.REAL; - double rightReal = rightNumeric.asDouble(); - return SQLite3Constant.createRealConstant(leftReal - rightReal); - } - } + return null; } }, @@ -1533,8 +1361,12 @@ public SQLite3Constant getExpectedValue() { if (left.getExpectedValue() == null || right.getExpectedValue() == null) { return null; } - return operation.applyOperand(left.getExpectedValue(), left.getAffinity(), right.getExpectedValue(), - right.getAffinity()); + SQLite3Constant result = operation.applyOperand(left.getExpectedValue(), left.getAffinity(), + right.getExpectedValue(), right.getAffinity()); + if (result != null && result.isReal()) { + SQLite3Cast.checkDoubleIsInsideDangerousRange(result.asDouble()); + } + return result; } public static Sqlite3BinaryOperation create(SQLite3Expression leftVal, SQLite3Expression rightVal, @@ -1627,17 +1459,25 @@ public static ConstantTuple applyAffinities(TypeAffinity leftAffinity, TypeAffin if (leftAffinity.isNumeric() && (rightAffinity == TypeAffinity.TEXT || rightAffinity == TypeAffinity.BLOB || rightAffinity == TypeAffinity.NONE)) { right = right.applyNumericAffinity(); + assert right != null; } else if (rightAffinity.isNumeric() && (leftAffinity == TypeAffinity.TEXT || leftAffinity == TypeAffinity.BLOB || leftAffinity == TypeAffinity.NONE)) { left = left.applyNumericAffinity(); + assert left != null; } // If one operand has TEXT affinity and the other has no affinity, then TEXT // affinity is applied to the other operand. if (leftAffinity == TypeAffinity.TEXT && rightAffinity == TypeAffinity.NONE) { right = right.applyTextAffinity(); + if (right == null) { + throw new IgnoreMeException(); + } } else if (rightAffinity == TypeAffinity.TEXT && leftAffinity == TypeAffinity.NONE) { left = left.applyTextAffinity(); + if (left == null) { + throw new IgnoreMeException(); + } } return new ConstantTuple(left, right); } @@ -1723,4 +1563,198 @@ public boolean omitBracketsWhenPrinting() { } } + public static class SQLite3WithClause extends SQLite3Expression { + + private final SQLite3Expression left; + private SQLite3Expression right; + + public SQLite3WithClause(SQLite3Expression left, SQLite3Expression right) { + this.left = left; + this.right = right; + } + + public SQLite3Expression getLeft() { + return this.left; + } + + public SQLite3Expression getRight() { + return this.right; + } + + public void updateRight(SQLite3Expression right) { + this.right = right; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3Alias extends SQLite3Expression { + + private final SQLite3Expression originalExpression; + private final SQLite3Expression aliasExpression; + + public SQLite3Alias(SQLite3Expression originalExpression, SQLite3Expression aliasExpression) { + this.originalExpression = originalExpression; + this.aliasExpression = aliasExpression; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + public SQLite3Expression getOriginalExpression() { + return originalExpression; + } + + public SQLite3Expression getAliasExpression() { + return aliasExpression; + } + } + + public static class SQLite3TableAndColumnRef extends SQLite3Expression { + + private final SQLite3Table table; + + public SQLite3TableAndColumnRef(SQLite3Table table) { + this.table = table; + } + + public SQLite3Table getTable() { + return this.table; + } + + public String getString() { + StringBuilder sb = new StringBuilder(); + sb.append(table.getName()); + sb.append("("); + Boolean isFirstColumn = true; + for (SQLite3Column c : this.table.getColumns()) { + if (!isFirstColumn) { + sb.append(", "); + } + sb.append(c.getName()); + isFirstColumn = false; + } + sb.append(")"); + return sb.toString(); + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3Values extends SQLite3Expression { + + private final Map> values; + private final List columns; + + public SQLite3Values(Map> values, List columns) { + this.values = values; + this.columns = columns; + } + + public Map> getValues() { + return this.values; + } + + public List getColumns() { + return this.columns; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + // The ExpressionBag is not a built-in SQL feature, + // but rather a utility class used in CODDTest's oracle construction + // to substitute expressions with their corresponding constant values. + public static class SQLite3ExpressionBag extends SQLite3Expression { + private SQLite3Expression innerExpr; + + public SQLite3ExpressionBag(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public void updateInnerExpr(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public SQLite3Expression getInnerExpr() { + return innerExpr; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } + + public static class SQLite3Typeof extends SQLite3Expression { + private final SQLite3Expression innerExpr; + + public SQLite3Typeof(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public SQLite3Expression getInnerExpr() { + return innerExpr; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } + + public static class SQLite3ResultMap extends SQLite3Expression { + private final SQLite3Values values; + private final List columns; + private final List summary; + private final SQLite3DataType summaryDataType; + + public SQLite3ResultMap(SQLite3Values values, List columns, List summary, + SQLite3DataType summaryDataType) { + this.values = values; + this.columns = columns; + this.summary = summary; + this.summaryDataType = summaryDataType; + + Map> vs = values.getValues(); + if (vs.get(vs.keySet().iterator().next()).size() != summary.size()) { + throw new AssertionError(); + } + } + + public SQLite3Values getValues() { + return this.values; + } + + public List getColumns() { + return this.columns; + } + + public List getSummary() { + return this.summary; + } + + public SQLite3DataType getSummaryDataType() { + return this.summaryDataType; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Function.java b/src/sqlancer/sqlite3/ast/SQLite3Function.java index 79f0e157b..07bd2ae75 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Function.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Function.java @@ -275,6 +275,8 @@ public SQLite3Constant apply(SQLite3Constant... args) { /** * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number * of arguments is returned. + * + * @return the number of arguments */ public int getNrArgs() { return nrArgs; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 7c5db143a..176057e8a 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -1,11 +1,19 @@ package sqlancer.sqlite3.ast; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import sqlancer.IgnoreMeException; +import sqlancer.common.ast.newast.Select; +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3Select extends SQLite3Expression { +public class SQLite3Select extends SQLite3Expression + implements Select { private SelectType fromOptions = SelectType.ALL; private List fromList = Collections.emptyList(); @@ -17,6 +25,27 @@ public class SQLite3Select extends SQLite3Expression { private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; + private SQLite3WithClause withClause; + + public SQLite3Select() { + } + + public SQLite3Select(SQLite3Select other) { + fromOptions = other.fromOptions; + fromList = new ArrayList<>(other.fromList); + whereClause = other.whereClause; + groupByClause = other.groupByClause; + limitClause = other.limitClause; + orderByClause = new ArrayList<>(other.orderByClause); + offsetClause = other.offsetClause; + fetchColumns = new ArrayList<>(other.fetchColumns); + joinStatements = new ArrayList<>(); + for (Join j : other.joinStatements) { + joinStatements.add(new Join(j)); + } + havingClause = other.havingClause; + withClause = other.withClause; + } public enum SelectType { DISTINCT, ALL; @@ -26,10 +55,6 @@ public void setSelectType(SelectType fromOptions) { this.setFromOptions(fromOptions); } - public void setFromTables(List fromTables) { - this.setFromList(fromTables); - } - public SelectType getFromOptions() { return fromOptions; } @@ -38,66 +63,82 @@ public void setFromOptions(SelectType fromOptions) { this.fromOptions = fromOptions; } + @Override public List getFromList() { return fromList; } + @Override public void setFromList(List fromList) { this.fromList = fromList; } + @Override public SQLite3Expression getWhereClause() { return whereClause; } + @Override public void setWhereClause(SQLite3Expression whereClause) { this.whereClause = whereClause; } + @Override public void setGroupByClause(List groupByClause) { this.groupByClause = groupByClause; } + @Override public List getGroupByClause() { return groupByClause; } + @Override public void setLimitClause(SQLite3Expression limitClause) { this.limitClause = limitClause; } + @Override public SQLite3Expression getLimitClause() { return limitClause; } - public List getOrderByClause() { + @Override + public List getOrderByClauses() { return orderByClause; } - public void setOrderByExpressions(List orderBy) { + @Override + public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } + @Override public void setOffsetClause(SQLite3Expression offsetClause) { this.offsetClause = offsetClause; } + @Override public SQLite3Expression getOffsetClause() { return offsetClause; } + @Override public void setFetchColumns(List fetchColumns) { this.fetchColumns = fetchColumns; } + @Override public List getFetchColumns() { return fetchColumns; } + @Override public void setJoinClauses(List joinStatements) { this.joinStatements = joinStatements; } + @Override public List getJoinClauses() { return joinStatements; } @@ -108,13 +149,50 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } + @Override public void setHavingClause(SQLite3Expression havingClause) { this.havingClause = havingClause; } + @Override public SQLite3Expression getHavingClause() { assert orderByClause != null; return havingClause; } + @Override + public String asString() { + return SQLite3Visitor.asString(this); + } + + public void setWithClause(SQLite3WithClause withClause) { + this.withClause = withClause; + } + + public void updateWithClauseRight(SQLite3Expression withClauseRight) { + this.withClause.updateRight(withClauseRight); + } + + public SQLite3Expression getWithClause() { + return this.withClause; + } + + // This method is used in CODDTest to test subquery by replacing a table name + // in the SELECT clause with a derived table expression. + public void replaceFromTable(String tableName, SQLite3Expression newFromExpression) { + int replaceIdx = -1; + for (int i = 0; i < fromList.size(); ++i) { + SQLite3Expression f = fromList.get(i); + if (f instanceof SQLite3TableReference) { + SQLite3TableReference tableRef = (SQLite3TableReference) f; + if (tableRef.getTable().getName().equals(tableName)) { + replaceIdx = i; + } + } + } + if (replaceIdx == -1) { + throw new IgnoreMeException(); + } + fromList.set(replaceIdx, newFromExpression); + } } diff --git a/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java b/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java index 319b874e0..0d5606c6a 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java +++ b/src/sqlancer/sqlite3/ast/SQLite3UnaryOperation.java @@ -3,10 +3,10 @@ import java.util.Optional; import sqlancer.Randomly; +import sqlancer.common.visitor.UnaryOperation; import sqlancer.sqlite3.SQLite3CollateHelper; import sqlancer.sqlite3.schema.SQLite3DataType; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; -import sqlancer.visitor.UnaryOperation; public class SQLite3UnaryOperation extends SQLite3Expression implements UnaryOperation { @@ -34,11 +34,9 @@ public SQLite3CollateSequence getImplicitCollateSequence() { } /** - * Supported unary prefix operators are these: + * Supported unary prefix operators are these: -, +, ~, and NOT. * - * - + ~ NOT - * - * @see https://www.sqlite.org/lang_expr.html + * @see SQL Language Expressions * */ public enum UnaryOperator { diff --git a/src/sqlancer/sqlite3/ast/SQLite3WindowFunction.java b/src/sqlancer/sqlite3/ast/SQLite3WindowFunction.java index e48b9c42f..917949198 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3WindowFunction.java +++ b/src/sqlancer/sqlite3/ast/SQLite3WindowFunction.java @@ -4,8 +4,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Provider; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; diff --git a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java index 35133a6c4..e795f610e 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3AnalyzeGenerator.java @@ -1,9 +1,9 @@ package sqlancer.sqlite3.gen; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; public final class SQLite3AnalyzeGenerator { @@ -14,7 +14,7 @@ private enum AnalyzeTarget { SCHEMA, TABLE, INDEX, SQL_MASTER } - public static Query generateAnalyze(SQLite3GlobalState globalState) { + public static SQLQueryAdapter generateAnalyze(SQLite3GlobalState globalState) { final StringBuilder sb = new StringBuilder("ANALYZE"); if (Randomly.getBoolean()) { sb.append(" "); @@ -35,7 +35,7 @@ public static Query generateAnalyze(SQLite3GlobalState globalState) { throw new AssertionError(); } } - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("The database file is locked")); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index 7defc9d29..4e83fbca5 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -4,7 +4,8 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; @@ -38,6 +39,10 @@ public boolean isContainsPrimaryKey() { } public String createColumn(String columnName, SQLite3GlobalState globalState, List columns) { + if (globalState.getDbmsSpecificOptions().oracles == SQLite3OracleFactory.PQS + || !globalState.getDbmsSpecificOptions().testCheckConstraints) { + allowCheck = false; + } sb.append(columnName); sb.append(" "); String dataType = Randomly.fromOptions("INT", "TEXT", "BLOB", "REAL", "INTEGER"); @@ -46,7 +51,7 @@ public String createColumn(String columnName, SQLite3GlobalState globalState, Li if (Randomly.getBooleanWithRatherLowProbability()) { List constraints = Randomly.subset(Constraints.values()); if (!Randomly.getBooleanWithSmallProbability() - || globalState.getDmbsSpecificOptions().testGeneratedColumns) { + || !globalState.getDbmsSpecificOptions().testGeneratedColumns) { constraints.remove(Constraints.GENERATED_AS); } if (constraints.contains(Constraints.GENERATED_AS)) { @@ -112,8 +117,8 @@ public String createColumn(String columnName, SQLite3GlobalState globalState, Li } } if (allowDefaultValue && Randomly.getBooleanWithSmallProbability()) { - sb.append(" DEFAULT " - + SQLite3Visitor.asString(SQLite3ExpressionGenerator.getRandomLiteralValue(globalState))); + sb.append(" DEFAULT "); + sb.append(SQLite3Visitor.asString(SQLite3ExpressionGenerator.getRandomLiteralValue(globalState))); } if (Randomly.getBooleanWithSmallProbability()) { String randomCollate = SQLite3Common.getRandomCollate(); diff --git a/src/sqlancer/sqlite3/gen/SQLite3Common.java b/src/sqlancer/sqlite3/gen/SQLite3Common.java index 46de2c35f..e40e4d524 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3Common.java +++ b/src/sqlancer/sqlite3/gen/SQLite3Common.java @@ -2,17 +2,12 @@ import java.util.ArrayList; import java.util.List; -import java.util.Optional; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; -import sqlancer.sqlite3.ast.SQLite3Cast; -import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; import sqlancer.sqlite3.schema.SQLite3DataType; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -28,43 +23,12 @@ public static String getRandomCollate() { return Randomly.fromOptions(" COLLATE BINARY", " COLLATE RTRIM", " COLLATE NOCASE"/* , " COLLATE UINT" */); } - public static String createTableName(int nr) { - return String.format("t%d", nr); - } - - public static String createColumnName(int nr) { - return String.format("c%d", nr); - } - - public static String createIndexName(int nr) { - return String.format("i%d", nr); - } - public static String getCheckConstraint(SQLite3GlobalState globalState, List columns) { SQLite3Expression expression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) .generateExpression(); return " CHECK ( " + SQLite3Visitor.asString(expression) + ")"; } - public static SQLite3Expression getTrueExpression(List columns, SQLite3GlobalState globalState) { - SQLite3Expression randomExpression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) - .generateExpression(); - SQLite3Constant expectedValue = randomExpression.getExpectedValue(); - if (expectedValue == null) { - throw new IgnoreMeException(); - } - Optional val = SQLite3Cast.isTrue(expectedValue); - if (!val.isPresent()) { - return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, randomExpression); - } - if (val.get()) { - return randomExpression; - } else { - return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_FALSE, randomExpression); - } - - } - // TODO: refactor others to use this method // https://www.sqlite.org/syntax/ordering-term.html public static String getOrderingTerm(List columns, SQLite3GlobalState globalState) { @@ -100,7 +64,7 @@ public static String getFreeTableName(SQLite3Schema s) { int nr = 0; String[] name = new String[1]; do { - name[0] = SQLite3Common.createTableName(nr++); + name[0] = DBMSCommon.createTableName(nr++); } while (s.getDatabaseTables().stream().anyMatch(tab -> tab.getName().contentEquals(name[0]))); return name[0]; } @@ -118,7 +82,7 @@ public static String getFreeIndexName(SQLite3Schema s) { List indexNames = s.getIndexNames(); String candidateName; do { - candidateName = SQLite3Common.createIndexName((int) Randomly.getNotCachedInteger(0, 100)); + candidateName = DBMSCommon.createIndexName((int) Randomly.getNotCachedInteger(0, 100)); } while (indexNames.contains(candidateName)); return candidateName; } @@ -127,7 +91,7 @@ public static String getFreeColumnName(SQLite3Table t) { List indexNames = t.getColumns(); final String[] candidateName = new String[1]; do { - candidateName[0] = SQLite3Common.createColumnName((int) Randomly.getNotCachedInteger(0, 100)); + candidateName[0] = DBMSCommon.createColumnName((int) Randomly.getNotCachedInteger(0, 100)); } while (indexNames.stream().anyMatch(c -> c.getName().contentEquals(candidateName[0]))); return candidateName[0]; } @@ -156,7 +120,7 @@ public static List getOrderBy(List columns, SQ } public static SQLite3Column createColumn(int i) { - return new SQLite3Column(createColumnName(i), SQLite3DataType.NONE, false, false, null); + return new SQLite3Column(DBMSCommon.createColumnName(i), SQLite3DataType.NONE, false, false, null); } public static List getTableRefs(List tables, SQLite3Schema s) { diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index c782eff41..61f821382 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -1,14 +1,14 @@ package sqlancer.sqlite3.gen; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; public final class SQLite3CreateVirtualRtreeTabelGenerator { @@ -16,8 +16,16 @@ public final class SQLite3CreateVirtualRtreeTabelGenerator { private SQLite3CreateVirtualRtreeTabelGenerator() { } - public static Query createTableStatement(String rTreeTableName, SQLite3GlobalState globalState) { - Set errors = new HashSet<>(); + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeRtreeTableName(), globalState); + } + + public static SQLQueryAdapter createTableStatement(String rTreeTableName, SQLite3GlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); List columns = new ArrayList<>(); StringBuilder sb = new StringBuilder("CREATE VIRTUAL TABLE "); sb.append(rTreeTableName); @@ -36,7 +44,7 @@ public static Query createTableStatement(String rTreeTableName, SQLite3GlobalSta for (int i = 0; i < Randomly.smallNumber(); i++) { sb.append(", "); sb.append("+"); - String columnName = SQLite3Common.createColumnName(size + i); + String columnName = DBMSCommon.createColumnName(size + i); SQLite3ColumnBuilder columnBuilder = new SQLite3ColumnBuilder().allowPrimaryKey(false).allowNotNull(false) .allowUnique(false).allowCheck(false); String c = columnBuilder.createColumn(columnName, globalState, columns); @@ -48,7 +56,7 @@ public static Query createTableStatement(String rTreeTableName, SQLite3GlobalSta errors.add("Wrong number of columns for an rtree table"); errors.add("Too many columns for an rtree table"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java index a54a83069..8c00b3231 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java @@ -1,20 +1,17 @@ package sqlancer.sqlite3.gen; -import java.sql.SQLException; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.sqlite3.SQLite3Provider.Action; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; public final class SQLite3ExplainGenerator { private SQLite3ExplainGenerator() { } - public static Query explain(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter explain(SQLite3GlobalState globalState) throws Exception { StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN "); if (Randomly.getBoolean()) { @@ -24,9 +21,16 @@ public static Query explain(SQLite3GlobalState globalState) throws SQLException do { action = Randomly.fromOptions(SQLite3Provider.Action.values()); } while (action == Action.EXPLAIN); - Query query = action.getQuery(globalState); + SQLQueryAdapter query = action.getQuery(globalState); sb.append(query); - return new QueryAdapter(sb.toString(), query.getExpectedErrors()); + return new SQLQueryAdapter(sb.toString(), query.getExpectedErrors()); + } + + public static String explain(String selectStr) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN QUERY PLAN "); + sb.append(selectStr); + return sb.toString(); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index db8600f16..e4f0741f9 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -7,7 +7,11 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; import sqlancer.sqlite3.ast.SQLite3Case.CasePair; @@ -28,13 +32,16 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; import sqlancer.sqlite3.ast.SQLite3Function.ComputableFunction; import sqlancer.sqlite3.ast.SQLite3RowValueExpression; +import sqlancer.sqlite3.ast.SQLite3Select; import sqlancer.sqlite3.ast.SQLite3UnaryOperation; import sqlancer.sqlite3.ast.SQLite3UnaryOperation.UnaryOperator; import sqlancer.sqlite3.oracle.SQLite3RandomQuerySynthesizer; @@ -43,12 +50,15 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3RowValue; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3ExpressionGenerator { +public class SQLite3ExpressionGenerator implements ExpressionGenerator, + NoRECGenerator, + TLPWhereGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; private boolean tryToGenerateKnownResult; private List columns = Collections.emptyList(); + private List targetTables; private final Randomly r; private boolean deterministicOnly; private boolean allowMatchClause; @@ -56,6 +66,20 @@ public class SQLite3ExpressionGenerator { private boolean allowSubqueries; private boolean allowAggreates; + public SQLite3ExpressionGenerator(SQLite3ExpressionGenerator other) { + this.rw = other.rw; + this.globalState = other.globalState; + this.tryToGenerateKnownResult = other.tryToGenerateKnownResult; + this.columns = new ArrayList<>(other.columns); + this.targetTables = other.targetTables; + this.r = other.r; + this.deterministicOnly = other.deterministicOnly; + this.allowMatchClause = other.allowMatchClause; + this.allowAggregateFunctions = other.allowAggregateFunctions; + this.allowSubqueries = other.allowSubqueries; + this.allowAggreates = other.allowAggreates; + } + private enum LiteralValueType { INTEGER, NUMERIC, STRING, BLOB_LITERAL, NULL } @@ -66,44 +90,52 @@ public SQLite3ExpressionGenerator(SQLite3GlobalState globalState) { } public SQLite3ExpressionGenerator deterministicOnly() { - this.deterministicOnly = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.deterministicOnly = true; + return gen; } public SQLite3ExpressionGenerator allowAggregateFunctions() { - this.allowAggregateFunctions = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.allowAggregateFunctions = true; + return gen; } public SQLite3ExpressionGenerator setColumns(List columns) { - this.columns = columns; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.columns = new ArrayList<>(columns); + return gen; } public SQLite3ExpressionGenerator setRowValue(SQLite3RowValue rw) { - this.rw = rw; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.rw = rw; + return gen; } public SQLite3ExpressionGenerator allowMatchClause() { - this.allowMatchClause = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.allowMatchClause = true; + return gen; } public SQLite3ExpressionGenerator allowSubqueries() { - this.allowSubqueries = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.allowSubqueries = true; + return gen; } public SQLite3ExpressionGenerator tryToGenerateKnownResult() { - this.tryToGenerateKnownResult = true; - return this; + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.tryToGenerateKnownResult = true; + return gen; } public static SQLite3Expression getRandomLiteralValue(SQLite3GlobalState globalState) { return new SQLite3ExpressionGenerator(globalState).getRandomLiteralValueInternal(globalState.getRandomly()); } + @Override public List generateOrderBys() { List expressions = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { @@ -114,22 +146,28 @@ public List generateOrderBys() { public List getRandomJoinClauses(List tables) { List joinStatements = new ArrayList<>(); - if (!globalState.getDmbsSpecificOptions().testJoins) { + if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } + List options = new ArrayList<>(Arrays.asList(JoinType.values())); if (Randomly.getBoolean() && tables.size() > 1) { int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } for (int i = 0; i < nrJoinClauses; i++) { SQLite3Expression joinClause = generateExpression(); SQLite3Table table = Randomly.fromList(tables); tables.remove(table); - JoinType options; - options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS, JoinType.OUTER, JoinType.NATURAL); - if (options == JoinType.NATURAL) { + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { // NATURAL joins do not have an ON clause joinClause = null; } - Join j = new SQLite3Expression.Join(table, joinClause, options); + Join j = new SQLite3Expression.Join(table, joinClause, selectedOption); joinStatements.add(j); } @@ -143,7 +181,7 @@ public SQLite3Expression generateOrderingTerm() { if (Randomly.getBoolean()) { expr = new SQLite3OrderingTerm(expr, Ordering.getRandomValue()); } - if (globalState.getDmbsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { expr = new SQLite3PostfixText(expr, Randomly.fromOptions(" NULLS FIRST", " NULLS LAST"), null /* expr.getExpectedValue() */) { @Override @@ -155,18 +193,15 @@ public boolean omitBracketsWhenPrinting() { return expr; } - /** - * - * @see https://www.sqlite.org/syntax/literal-value.html - * - * @return + /* + * https://www.sqlite.org/syntax/literal-value.html */ private SQLite3Expression getRandomLiteralValueInternal(Randomly r) { LiteralValueType randomLiteral = Randomly.fromOptions(LiteralValueType.values()); switch (randomLiteral) { case INTEGER: if (Randomly.getBoolean()) { - return SQLite3Constant.createIntConstant(r.getInteger()); + return SQLite3Constant.createIntConstant(r.getInteger(), Randomly.getBoolean()); } else { return SQLite3Constant.createTextConstant(String.valueOf(r.getInteger())); } @@ -234,12 +269,15 @@ public SQLite3Expression getRandomExpression(int depth) { if (!allowSubqueries) { list.remove(ExpressionType.RANDOM_QUERY); } - if (!globalState.getDmbsSpecificOptions().testFunctions) { + if (!globalState.getDbmsSpecificOptions().testFunctions) { list.remove(ExpressionType.FUNCTION); } - if (!globalState.getDmbsSpecificOptions().testMatch) { + if (!globalState.getDbmsSpecificOptions().testMatch) { list.remove(ExpressionType.MATCH); } + if (!globalState.getDbmsSpecificOptions().testIn) { + list.remove(ExpressionType.IN_OPERATOR); + } ExpressionType randomExpressionType = Randomly.fromList(list); switch (randomExpressionType) { case AND_OR_CHAIN: @@ -325,7 +363,7 @@ private enum RowValueComparison { STANDARD_COMPARISON, BETWEEN, IN } - /** + /* * https://www.sqlite.org/rowvalue.html */ private SQLite3Expression getRowValueComparison(int depth) { @@ -482,7 +520,7 @@ static AnyFunction getRandom(SQLite3GlobalState globalState) { private static List getAllFunctions(SQLite3GlobalState globalState) { List functions = new ArrayList<>(Arrays.asList(AnyFunction.values())); - if (!globalState.getDmbsSpecificOptions().testSoundex) { + if (!globalState.getDbmsSpecificOptions().testSoundex) { boolean removed = functions.removeIf(f -> f.name.equals("soundex")); if (!removed) { throw new IllegalStateException(); @@ -525,6 +563,12 @@ private SQLite3Expression getFunction(SQLite3GlobalState globalState, int depth) nrArgs += Randomly.smallNumber(); } List expressions = randomFunction.generateArguments(nrArgs, depth + 1, this); + // The second argument of LIKELIHOOD must be a float number within 0.0 -1.0 + if (randomFunction == AnyFunction.LIKELIHOOD) { + SQLite3Expression lastArg = SQLite3Constant.createRealConstant(Randomly.getPercentage()); + expressions.remove(expressions.size() - 1); + expressions.add(lastArg); + } return new SQLite3Expression.Function(randomFunction.toString(), expressions.toArray(new SQLite3Expression[0])); } @@ -582,6 +626,11 @@ private SQLite3Expression getComputableFunction(int depth) { args[i] = new SQLite3Distinct(args[i]); } } + // The second argument of LIKELIHOOD must be a float number within 0.0 -1.0 + if (func == ComputableFunction.LIKELIHOOD) { + SQLite3Expression lastArg = SQLite3Constant.createRealConstant(Randomly.getPercentage()); + args[args.length - 1] = lastArg; + } return new SQLite3Function(func, args); } @@ -640,4 +689,105 @@ public SQLite3Expression getHavingClause() { return generateExpression(); } + @Override + public SQLite3Expression generatePredicate() { + return generateExpression(); + } + + @Override + public SQLite3Expression negatePredicate(SQLite3Expression predicate) { + return new SQLite3UnaryOperation(UnaryOperator.NOT, predicate); + } + + @Override + public SQLite3Expression isNull(SQLite3Expression expr) { + return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); + } + + public SQLite3Expression generateResultKnownExpression() { + SQLite3Expression expr; + do { + expr = generateExpression(); + } while (expr.getExpectedValue() == null); + return expr; + } + + @Override + public SQLite3ExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.targetTables = targetTables.getTables(); + gen.columns = targetTables.getColumns(); + return gen; + } + + @Override + public SQLite3Expression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public SQLite3Select generateSelect() { + return new SQLite3Select(); + } + + @Override + public List getRandomJoinClauses() { + return getRandomJoinClauses(targetTables); + } + + @Override + public List getTableRefs() { + List tableRefs = new ArrayList<>(); + for (SQLite3Table t : targetTables) { + SQLite3TableReference tableRef; + if (Randomly.getBooleanWithSmallProbability() && !globalState.getSchema().getIndexNames().isEmpty()) { + tableRef = new SQLite3TableReference(globalState.getSchema().getRandomIndexOrBailout(), t); + } else { + tableRef = new SQLite3TableReference(t); + } + tableRefs.add(tableRef); + } + return tableRefs; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + List columns = new ArrayList<>(); + if (shouldCreateDummy && Randomly.getBoolean()) { + columns.add(new SQLite3ColumnName(SQLite3Column.createDummy("*"), null)); + } else { + columns = Randomly.nonEmptySubset(this.columns).stream().map(c -> new SQLite3ColumnName(c, null)) + .collect(Collectors.toList()); + } + return columns; + } + + @Override + public String generateOptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition, + boolean shouldUseAggregate) { + if (Randomly.getBoolean()) { + select.setOrderByClauses(generateOrderBys()); + } + if (shouldUseAggregate) { + select.setFetchColumns(Arrays.asList(new SQLite3Aggregate(Collections.emptyList(), + SQLite3Aggregate.SQLite3AggregateFunction.COUNT_ALL))); + } else { + SQLite3ColumnName aggr = new SQLite3ColumnName(SQLite3Column.createDummy("*"), null); + select.setFetchColumns(Arrays.asList(aggr)); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition) { + SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, + whereCondition); + SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ")"; + } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3MatchStringGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3MatchStringGenerator.java index aa70b4e84..5f68b71ad 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3MatchStringGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3MatchStringGenerator.java @@ -82,7 +82,8 @@ private void appendColspec() { if (i != 0) { sb.append(", "); } - sb.append("c" + Randomly.smallNumber()); + sb.append("c"); + sb.append(Randomly.smallNumber()); } if (braces) { sb.append("}"); diff --git a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java index fa2c5d019..c83cf4958 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3PragmaGenerator.java @@ -1,13 +1,12 @@ package sqlancer.sqlite3.gen; import java.sql.SQLException; -import java.util.HashSet; -import java.util.Set; import java.util.function.Supplier; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; public class SQLite3PragmaGenerator { @@ -47,20 +46,10 @@ private enum Pragma { WAL_CHECKPOINT; // // WRITEABLE_SCHEMA - // VDBE_ADDOPTRACE(PragmaAttribute.DEBUG); // produces too much textual output directly on the console - // VDBE_LISTING(PragmaAttribute.DEBUG); // produces too much textual output directly on the console - - Pragma(PragmaAttribute... attrs) { - } - - private enum PragmaAttribute { - DEBUG /* only available in debug mode */ - } - } private final StringBuilder sb = new StringBuilder(); - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); public void createPragma(String pragmaName, Supplier supplier) { boolean setSchema = Randomly.getBoolean(); @@ -79,7 +68,7 @@ public void createPragma(String pragmaName, Supplier supplier) { } } - public QueryAdapter insert(SQLite3GlobalState globalState) { + public SQLQueryAdapter insert(SQLite3GlobalState globalState) { Randomly r = globalState.getRandomly(); Pragma p = Randomly.fromOptions(Pragma.values()); switch (p) { @@ -154,18 +143,7 @@ public QueryAdapter insert(SQLite3GlobalState globalState) { } break; case INTEGRITY_CHECK: - // errors.add("malformed JSON"); - // errors.add("JSON cannot hold BLOB values"); - // errors.add("json_object() labels must be TEXT"); - // errors.add("requires an even number of arguments"); - // errors.add("needs an odd number of arguments"); - // errors.add("overflow"); - // errors.add("JSON path error"); - if (Randomly.getBoolean()) { - createPragma("integrity_check", () -> null); - } else { - sb.append(String.format("PRAGMA integrity_check(%d)", r.getInteger())); - } + createPragma("integrity_check", () -> null); break; case JOURNAL_MODE: // OFF is no longer generated, since it might corrupt the database upon failed @@ -242,11 +220,11 @@ public QueryAdapter insert(SQLite3GlobalState globalState) { } sb.append(";"); String pragmaString = sb.toString(); - // errors.add("cannot change"); - return new QueryAdapter(pragmaString, errors); + errors.add("The database file is locked"); + return new SQLQueryAdapter(pragmaString, errors); } - public static QueryAdapter insertPragma(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter insertPragma(SQLite3GlobalState globalState) throws SQLException { return new SQLite3PragmaGenerator().insert(globalState); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java index 44b0fa86c..b3359b15d 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ReindexGenerator.java @@ -1,16 +1,13 @@ package sqlancer.sqlite3.gen; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; /** - * @see https://www.sqlite.org/lang_reindex.html + * @see REINDEX */ public final class SQLite3ReindexGenerator { @@ -21,10 +18,11 @@ private enum Target { TABLE, INDEX, COLLATION_NAME } - public static Query executeReindex(SQLite3GlobalState globalState) { + public static SQLQueryAdapter executeReindex(SQLite3GlobalState globalState) { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder("REINDEX"); - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("The database file is locked"); Target t = Randomly.fromOptions(Target.values()); if (Randomly.getBoolean()) { sb.append(" "); @@ -45,6 +43,6 @@ public static Query executeReindex(SQLite3GlobalState globalState) { throw new AssertionError(t); } } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java index e7ee11d1d..eff3a83af 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3TransactionGenerator.java @@ -1,40 +1,40 @@ package sqlancer.sqlite3.gen; -import java.util.Arrays; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; public final class SQLite3TransactionGenerator { private SQLite3TransactionGenerator() { } - public static Query generateCommit(SQLite3GlobalState globalState) { + public static SQLQueryAdapter generateCommit(SQLite3GlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append(Randomly.fromOptions("COMMIT", "END")); if (Randomly.getBoolean()) { sb.append(" TRANSACTION"); } - return new QueryAdapter(sb.toString(), - Arrays.asList("no transaction is active", "FOREIGN KEY constraint failed"), true); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("no transaction is active", + "The database file is locked", "FOREIGN KEY constraint failed"), true); } - public static Query generateBeginTransaction(SQLite3GlobalState globalState) { + public static SQLQueryAdapter generateBeginTransaction(SQLite3GlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("BEGIN "); if (Randomly.getBoolean()) { sb.append(Randomly.fromOptions("DEFERRED", "IMMEDIATE", "EXCLUSIVE")); } sb.append(" TRANSACTION;"); - return new QueryAdapter(sb.toString(), Arrays.asList("cannot start a transaction within a transaction")); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot start a transaction within a transaction", "The database file is locked")); } - public static Query generateRollbackTransaction(SQLite3GlobalState globalState) { + public static SQLQueryAdapter generateRollbackTransaction(SQLite3GlobalState globalState) { // TODO: could be extended by savepoint - return new QueryAdapter("ROLLBACK TRANSACTION;", Arrays.asList("no transaction is active"), true); + return new SQLQueryAdapter("ROLLBACK TRANSACTION;", + ExpectedErrors.from("no transaction is active", "The database file is locked"), true); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java index a2fc69566..36b129cd6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VacuumGenerator.java @@ -1,27 +1,26 @@ package sqlancer.sqlite3.gen; -import java.util.Arrays; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; /** - * @see https://www.sqlite.org/lang_vacuum.html + * @see VACUUM */ public final class SQLite3VacuumGenerator { private SQLite3VacuumGenerator() { } - public static Query executeVacuum(SQLite3GlobalState globalState) { + public static SQLQueryAdapter executeVacuum(SQLite3GlobalState globalState) { StringBuilder sb = new StringBuilder("VACUUM"); if (Randomly.getBoolean()) { sb.append(" "); sb.append(Randomly.fromOptions("temp", "main")); } - return new QueryAdapter(sb.toString(), Arrays.asList("cannot VACUUM from within a transaction")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot VACUUM from within a transaction", + "cannot VACUUM - SQL statements in progress", "The database file is locked")); } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java index b20f9ca04..6efb6aeff 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3VirtualFTSTableCommandGenerator.java @@ -1,12 +1,9 @@ package sqlancer.sqlite3.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; @@ -15,9 +12,9 @@ public class SQLite3VirtualFTSTableCommandGenerator { private final StringBuilder sb = new StringBuilder(); private final SQLite3Schema s; private final Randomly r; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); - public static Query create(SQLite3GlobalState globalState) { + public static SQLQueryAdapter create(SQLite3GlobalState globalState) { return new SQLite3VirtualFTSTableCommandGenerator(globalState.getSchema(), globalState.getRandomly()) .generate(); } @@ -31,7 +28,7 @@ private enum Action { AUTOMERGE, CRISISMERGE, INTEGRITYCHECK, MERGE, OPTIMIZE, REBUILD, USER_MERGE, PGSZ, RANK; // TODO: delete all } - private Query generate() { + private SQLQueryAdapter generate() { errors.add("has no column named rank"); SQLite3Table vTable = s.getRandomTableOrBailout(t -> t.isVirtual() && t.getName().startsWith("vt")); Action a = Randomly.fromOptions(Action.values()); @@ -93,7 +90,8 @@ private Query generate() { default: throw new AssertionError(); } - return new QueryAdapter(sb.toString(), errors); + errors.add("The database file is locked"); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java index 9dd8de8c7..5cf4e9339 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3AlterTable.java @@ -1,13 +1,11 @@ package sqlancer.sqlite3.gen.ddl; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -19,7 +17,7 @@ public class SQLite3AlterTable { private final StringBuilder sb = new StringBuilder(); private final SQLite3GlobalState globalState; - public static Query alterTable(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter alterTable(SQLite3GlobalState globalState) throws SQLException { SQLite3AlterTable alterTable = new SQLite3AlterTable(globalState); return alterTable.getQuery(globalState.getSchema(), alterTable); } @@ -32,8 +30,8 @@ public SQLite3AlterTable(SQLite3GlobalState globalState) { this.globalState = globalState; } - private Query getQuery(SQLite3Schema s, SQLite3AlterTable alterTable) throws AssertionError { - List errors = new ArrayList<>(); + private SQLQueryAdapter getQuery(SQLite3Schema s, SQLite3AlterTable alterTable) throws AssertionError { + ExpectedErrors errors = new ExpectedErrors(); errors.add("error in view"); errors.add("no such column"); // trigger errors.add("error in trigger"); // trigger @@ -82,7 +80,7 @@ private Query getQuery(SQLite3Schema s, SQLite3AlterTable alterTable) throws Ass default: throw new AssertionError(); } - return new QueryAdapter(alterTable.sb.toString(), errors, true); + return new SQLQueryAdapter(alterTable.sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index fe789c469..1e5254948 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -2,14 +2,13 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.gen.dml.SQLite3DeleteGenerator; @@ -31,7 +30,7 @@ private enum TriggerAction { INSERT, DELETE, UPDATE, RAISE } - public static Query create(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter create(SQLite3GlobalState globalState) throws SQLException { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder(); SQLite3Table table = s.getRandomTableOrBailout(t -> !t.isVirtual()); @@ -42,7 +41,8 @@ public static Query create(SQLite3GlobalState globalState) throws SQLException { } sb.append(" TRIGGER"); sb.append(" IF NOT EXISTS "); - sb.append("tr" + Randomly.smallNumber()); + sb.append("tr"); + sb.append(Randomly.smallNumber()); sb.append(" "); if (table.isView()) { sb.append("INSTEAD OF"); @@ -100,16 +100,16 @@ public static Query create(SQLite3GlobalState globalState) throws SQLException { sb.append(", 'asdf'"); } sb.append(")"); + sb.append(";"); break; default: throw new AssertionError(); } - sb.append(";"); } sb.append("END"); - return new QueryAdapter(sb.toString(), - Arrays.asList("parser stack overflow", "unsupported frame specification")); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("parser stack overflow", "unsupported frame specification")); } private static void appendTableNameAndWhen(SQLite3GlobalState globalState, StringBuilder sb, SQLite3Table table) { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index 5afcbc415..3ae9d16d5 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -4,10 +4,12 @@ import java.util.Arrays; import java.util.List; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.sqlite3.gen.SQLite3Common; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; public class SQLite3CreateVirtualFTSTableGenerator { @@ -20,7 +22,15 @@ public SQLite3CreateVirtualFTSTableGenerator(String tableName, Randomly r) { this.r = r; } - public static Query createTableStatement(String tableName, Randomly r) { + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeVirtualTableName(), globalState.getRandomly()); + } + + public static SQLQueryAdapter createTableStatement(String tableName, Randomly r) { return new SQLite3CreateVirtualFTSTableGenerator(tableName, r).create(); } @@ -36,7 +46,7 @@ private enum Fts4Options { MATCHINFO, TOKENIZE, PREFIX, ORDER, LANGUAGEID, COMPRESS, NOT_INDEXED } - public Query create() { + public SQLQueryAdapter create() { sb.append("CREATE VIRTUAL TABLE "); sb.append(tableName); sb.append(" USING "); @@ -45,7 +55,8 @@ public Query create() { } else { createFts5Table(); } - return new QueryAdapter(sb.toString(), Arrays.asList("unrecognized parameter", "unknown tokenizer: ascii")); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("unrecognized parameter", "unknown tokenizer: ascii"), true); } private void createFts4Table() { @@ -173,7 +184,7 @@ private void createTable(String ftsVersion, Ac columnAction, Ac tableAction) { if (i != 0) { sb.append(", "); } - sb.append(SQLite3Common.createColumnName(i)); + sb.append(DBMSCommon.createColumnName(i)); columnAction.action(); } tableAction.action(); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java index 043d0dcbb..3f4427bce 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropIndexGenerator.java @@ -1,11 +1,9 @@ package sqlancer.sqlite3.gen.ddl; -import java.util.Arrays; - -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; // see https://www.sqlite.org/lang_dropindex.html public final class SQLite3DropIndexGenerator { @@ -13,7 +11,7 @@ public final class SQLite3DropIndexGenerator { private SQLite3DropIndexGenerator() { } - public static Query dropIndex(SQLite3GlobalState globalState) { + public static SQLQueryAdapter dropIndex(SQLite3GlobalState globalState) { String indexName = globalState.getSchema().getRandomIndexOrBailout(); StringBuilder sb = new StringBuilder(); sb.append("DROP INDEX "); @@ -23,7 +21,7 @@ public static Query dropIndex(SQLite3GlobalState globalState) { sb.append('"'); sb.append(indexName); sb.append('"'); - return new QueryAdapter(sb.toString(), Arrays.asList( + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from( "[SQLITE_ERROR] SQL error or missing database (index associated with UNIQUE or PRIMARY KEY constraint cannot be dropped)"), true); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java index abb973057..ed5eb16e4 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3DropTableGenerator.java @@ -1,19 +1,17 @@ package sqlancer.sqlite3.gen.ddl; -import java.util.Arrays; - import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; public final class SQLite3DropTableGenerator { private SQLite3DropTableGenerator() { } - public static Query dropTable(SQLite3GlobalState globalState) { + public static SQLQueryAdapter dropTable(SQLite3GlobalState globalState) { if (globalState.getSchema().getTables(t -> !t.isView()).size() == 1) { throw new IgnoreMeException(); } @@ -22,8 +20,8 @@ public static Query dropTable(SQLite3GlobalState globalState) { sb.append("IF EXISTS "); } sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); - return new QueryAdapter(sb.toString(), - Arrays.asList("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", "Abort due to constraint violation (FOREIGN KEY constraint failed)", "SQL error or missing database"), true); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 6277ff653..c81d895ac 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -1,16 +1,15 @@ package sqlancer.sqlite3.gen.ddl; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Provider; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.gen.SQLite3Common; @@ -21,10 +20,13 @@ // see https://www.sqlite.org/lang_createindex.html public class SQLite3IndexGenerator { - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private final SQLite3GlobalState globalState; - public static Query insertIndex(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter insertIndex(SQLite3GlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexNames().size() >= globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } return new SQLite3IndexGenerator(globalState).create(); } @@ -32,7 +34,7 @@ public SQLite3IndexGenerator(SQLite3GlobalState globalState) throws SQLException this.globalState = globalState; } - private Query create() throws SQLException { + private SQLQueryAdapter create() throws SQLException { SQLite3Table t = globalState.getSchema() .getRandomTableOrBailout(tab -> !tab.isView() && !tab.isVirtual() && !tab.isReadOnly()); String q = createIndex(t, t.getColumns()); @@ -45,6 +47,7 @@ private Query create() throws SQLException { errors.add("non-deterministic use of julianday() in an index"); errors.add("non-deterministic use of date() in an index"); errors.add("non-deterministic use of datetime() in an index"); + errors.add("The database file is locked"); SQLite3Errors.addExpectedExpressionErrors(errors); if (!SQLite3Provider.mustKnowResult) { // can only happen when PRAGMA case_sensitive_like=ON; @@ -58,23 +61,26 @@ private Query create() throws SQLException { * https://www.mail-archive.com/sqlite-users@mailinglists.sqlite.org/msg115014.html). */ errors.add("[SQLITE_ERROR] SQL error or missing database (no such column:"); - return new QueryAdapter(q, errors, true); + return new SQLQueryAdapter(q, errors, true); } private String createIndex(SQLite3Table t, List columns) { StringBuilder sb = new StringBuilder(); sb.append("CREATE"); if (Randomly.getBoolean()) { - errors.add("[SQLITE_CONSTRAINT] Abort due to constraint violation (UNIQUE constraint failed"); + errors.add("UNIQUE constraint failed "); sb.append(" UNIQUE"); } sb.append(" INDEX"); if (Randomly.getBoolean()) { sb.append(" IF NOT EXISTS"); + } else { + errors.add("already exists"); } - sb.append(" " + SQLite3Common.getFreeIndexName(globalState.getSchema())); - sb.append(" ON"); - sb.append(" " + t.getName()); + sb.append(" "); + sb.append(SQLite3Common.getFreeIndexName(globalState.getSchema())); + sb.append(" ON "); + sb.append(t.getName()); sb.append("("); for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -104,8 +110,8 @@ private String createIndex(SQLite3Table t, List columns) { return sb.toString(); } - /** - * Appends ASC, DESC, or nothing + /* + * Appends ASC, DESC, or nothing. */ private void appendPotentialOrdering(StringBuilder sb) { if (Randomly.getBoolean()) { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index 1bafef41f..7847208a3 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -5,11 +5,14 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -44,10 +47,18 @@ public SQLite3TableGenerator(String tableName, SQLite3GlobalState globalState) { this.existingSchema = globalState.getSchema(); } - public static Query createTableStatement(String tableName, SQLite3GlobalState globalState) { + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeTableName(), globalState); + } + + public static SQLQueryAdapter createTableStatement(String tableName, SQLite3GlobalState globalState) { SQLite3TableGenerator sqLite3TableGenerator = new SQLite3TableGenerator(tableName, globalState); sqLite3TableGenerator.start(); - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); SQLite3Errors.addTableManipulationErrors(errors); errors.add("second argument to likelihood() must be a constant between 0.0 and 1.0"); errors.add("non-deterministic functions prohibited in generated columns"); @@ -55,12 +66,12 @@ public static Query createTableStatement(String tableName, SQLite3GlobalState gl errors.add("parser stack overflow"); errors.add("malformed JSON"); errors.add("JSON cannot hold BLOB values"); - return new QueryAdapter(sqLite3TableGenerator.sb.toString(), errors); + return new SQLQueryAdapter(sqLite3TableGenerator.sb.toString(), errors, true); } public void start() { sb.append("CREATE "); - if (globalState.getDmbsSpecificOptions().testTempTables && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testTempTables && Randomly.getBoolean()) { tempTable = true; if (Randomly.getBoolean()) { sb.append("TEMP "); @@ -72,18 +83,18 @@ public void start() { if (Randomly.getBoolean()) { sb.append("IF NOT EXISTS "); } - sb.append(tableName + " "); - sb.append("("); + sb.append(tableName); + sb.append(" ("); boolean allowPrimaryKeyInColumn = Randomly.getBoolean(); int nrColumns = 1 + Randomly.smallNumber(); for (int i = 0; i < nrColumns; i++) { - columns.add(SQLite3Column.createDummy(SQLite3Common.createColumnName(i))); + columns.add(SQLite3Column.createDummy(DBMSCommon.createColumnName(i))); } for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } - String columnName = SQLite3Common.createColumnName(columnId); + String columnName = DBMSCommon.createColumnName(columnId); SQLite3ColumnBuilder columnBuilder = new SQLite3ColumnBuilder() .allowPrimaryKey(allowPrimaryKeyInColumn && !containsPrimaryKey); sb.append(columnBuilder.createColumn(columnName, globalState, columns)); @@ -108,17 +119,23 @@ public void start() { } } - if (globalState.getDmbsSpecificOptions().testForeignKeys && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().testForeignKeys && Randomly.getBooleanWithSmallProbability()) { addForeignKey(); } - if (globalState.getDmbsSpecificOptions().testCheckConstraints + if (globalState.getDbmsSpecificOptions().testCheckConstraints && globalState + .getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS /* + * we are currently lacking a parser to + * read column definitions, and would + * interpret a COLLATE in the check + * constraint as belonging to the column + */ && Randomly.getBooleanWithRatherLowProbability()) { sb.append(SQLite3Common.getCheckConstraint(globalState, columns)); } sb.append(")"); - if (globalState.getDmbsSpecificOptions().testWithoutRowids && containsPrimaryKey && !containsAutoIncrement + if (globalState.getDbmsSpecificOptions().testWithoutRowids && containsPrimaryKey && !containsAutoIncrement && Randomly.getBoolean()) { // see https://sqlite.org/withoutrowid.html sb.append(" WITHOUT ROWID"); @@ -143,8 +160,8 @@ private void addColumnConstraints(String s) { * @see https://www.sqlite.org/foreignkeys.html */ private void addForeignKey() { - assert globalState.getDmbsSpecificOptions().testForeignKeys; - List foreignKeyColumns = new ArrayList<>(); + assert globalState.getDbmsSpecificOptions().testForeignKeys; + List foreignKeyColumns; if (Randomly.getBoolean()) { foreignKeyColumns = Arrays.asList(Randomly.fromList(columnNames)); } else { diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index ccaf22106..bc605a015 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -1,20 +1,19 @@ package sqlancer.sqlite3.gen.ddl; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.ast.SQLite3Select.SelectType; import sqlancer.sqlite3.gen.SQLite3Common; -import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; import sqlancer.sqlite3.oracle.SQLite3RandomQuerySynthesizer; import sqlancer.sqlite3.schema.SQLite3Schema; @@ -23,17 +22,21 @@ public final class SQLite3ViewGenerator { private SQLite3ViewGenerator() { } - public static Query dropView(SQLite3GlobalState globalState) { + public static SQLQueryAdapter dropView(SQLite3GlobalState globalState) { SQLite3Schema s = globalState.getSchema(); StringBuilder sb = new StringBuilder("DROP VIEW "); sb.append(s.getRandomViewOrBailout().getName()); - return new QueryAdapter(sb.toString(), true); + return new SQLQueryAdapter(sb.toString(), true); } - public static Query generate(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter generate(SQLite3GlobalState globalState) throws SQLException { + if (globalState.getSchema().getTables().getTables() + .size() >= globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } StringBuilder sb = new StringBuilder(); sb.append("CREATE"); - if (Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testTempTables && Randomly.getBoolean()) { sb.append(" "); sb.append(Randomly.fromOptions("TEMP", "TEMPORARY")); } @@ -42,39 +45,46 @@ public static Query generate(SQLite3GlobalState globalState) throws SQLException sb.append(" IF NOT EXISTS "); } sb.append(SQLite3Common.getFreeViewName(globalState.getSchema())); - List errors = new ArrayList<>(); + ExpectedErrors errors = new ExpectedErrors(); + SQLite3Errors.addExpectedExpressionErrors(errors); errors.add("is circularly defined"); errors.add("unsupported frame specification"); - if (Randomly.getBoolean()) { - SQLite3PivotedQuerySynthesisOracle queryGen = new SQLite3PivotedQuerySynthesisOracle(globalState); - try { - SQLite3Select q = queryGen.getQuery(globalState); - // for (SQLite3Expression expr : q.getFetchColumns()) { - // if (expr.getAffinity() != null || expr.getImplicitCollateSequence() != null || - // expr.getExplicitCollateSequence() != null) { - // throw new IgnoreMeException(); - // } - // } - if (!globalState.getDmbsSpecificOptions().testDistinctInView) { - q.setSelectType(SelectType.ALL); + errors.add("The database file is locked"); + int size = 1 + Randomly.smallNumber(); + columnNamesAs(sb, size); + SQLite3Expression randomQuery; + do { + randomQuery = SQLite3RandomQuerySynthesizer.generate(globalState, size); + } while (globalState.getDbmsSpecificOptions().oracles == SQLite3OracleFactory.PQS + && !checkAffinity(randomQuery)); + sb.append(SQLite3Visitor.asString(randomQuery)); + return new SQLQueryAdapter(sb.toString(), errors, true); + + } + + /** + * The affinity of columns in a view cannot be determined using features of the DBMS - this would need to be parsed + * from the CREATE TABLE and CREATE VIEW statements. This is non-trivial, and currently not implemented. Rather, we + * avoid generating expressions with an affinity or view. + * + * @see http://sqlite.1065341.n5.nabble.com/Determining-column-collating-functions-td108157.html#a108159 + * + * @param randomQuery + * + * @return true if the query can be used for PQS + */ + private static boolean checkAffinity(SQLite3Expression randomQuery) { + if (randomQuery instanceof SQLite3Select) { + for (SQLite3Expression expr : ((SQLite3Select) randomQuery).getFetchColumns()) { + if (expr.getExpectedValue() == null || expr.getAffinity() != null + || expr.getImplicitCollateSequence() != null || expr.getExplicitCollateSequence() != null) { + return false; } - int size = q.getFetchColumns().size(); - columnNamesAs(sb, size); - sb.append(SQLite3Visitor.asString(q)); - SQLite3PivotedQuerySynthesisOracle.addExpectedErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); - } catch (AssertionError e) { - throw new IgnoreMeException(); } + return true; } else { - int size = 1 + Randomly.smallNumber(); - columnNamesAs(sb, size); - SQLite3Expression randomQuery = SQLite3RandomQuerySynthesizer.generate(globalState, size); - sb.append(SQLite3Visitor.asString(randomQuery)); - SQLite3PivotedQuerySynthesisOracle.addExpectedErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); + return false; // the columns in UNION clauses can also have affinities } - } private static void columnNamesAs(StringBuilder sb, int size) { @@ -83,7 +93,7 @@ private static void columnNamesAs(StringBuilder sb, int size) { if (i != 0) { sb.append(", "); } - sb.append(SQLite3Common.createColumnName(i)); + sb.append(DBMSCommon.createColumnName(i)); } sb.append(")"); sb.append(" AS "); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 42982a1ce..a19da484f 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -1,38 +1,43 @@ package sqlancer.sqlite3.gen.dml; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public final class SQLite3DeleteGenerator { +public final class SQLite3DeleteGenerator extends AbstractDeleteGenerator { - private SQLite3DeleteGenerator() { + private final SQLite3GlobalState globalState; + private final SQLite3Table table; + + private SQLite3DeleteGenerator(SQLite3GlobalState globalState, SQLite3Table table) { + this.globalState = globalState; + this.table = table; + this.canAffectSchema = true; + } + + public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState) { + SQLite3Table table = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); + return deleteContent(globalState, table); } - public static Query deleteContent(SQLite3GlobalState globalState) { - SQLite3Table tableName = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); - return deleteContent(globalState, tableName); + public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLite3Table table) { + return new SQLite3DeleteGenerator(globalState, table).getStatement(); } - public static Query deleteContent(SQLite3GlobalState globalState, SQLite3Table tableName) { - StringBuilder sb = new StringBuilder(); - sb.append("DELETE FROM "); - sb.append(tableName.getName()); + @Override + public void buildStatement() { + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(SQLite3Visitor.asString(new SQLite3ExpressionGenerator(globalState) - .setColumns(tableName.getColumns()).generateExpression())); + appendWhereClause(SQLite3Visitor.asString( + new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } - List errors = new ArrayList<>(); SQLite3Errors.addExpectedExpressionErrors(errors); errors.addAll(Arrays.asList("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", "[SQLITE_CONSTRAINT] Abort due to constraint violation ", @@ -40,9 +45,8 @@ public static Query deleteContent(SQLite3GlobalState globalState, SQLite3Table t "[SQLITE_ERROR] SQL error or missing database (no such table:", "no such column", "too many levels of trigger recursion", "cannot UPDATE generated column", "cannot INSERT into generated column", "A table in the database is locked", - "load_extension() prohibited in triggers and views")); + "load_extension() prohibited in triggers and views", "The database file is locked")); SQLite3Errors.addDeleteErrors(errors); - return new QueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java index d7bb53e24..0e01cdc5a 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3InsertGenerator.java @@ -1,15 +1,14 @@ package sqlancer.sqlite3.gen.dml; import java.sql.SQLException; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3ToStringVisitor; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Constant; @@ -21,43 +20,33 @@ public class SQLite3InsertGenerator { private final Randomly r; - private final List errors; + private final ExpectedErrors errors; private final SQLite3GlobalState globalState; public SQLite3InsertGenerator(SQLite3GlobalState globalState, Randomly r) { this.globalState = globalState; this.r = r; - errors = new ArrayList<>(); + errors = new ExpectedErrors(); } - public static Query insertRow(SQLite3GlobalState globalState) throws SQLException { + public static SQLQueryAdapter insertRow(SQLite3GlobalState globalState) throws SQLException { SQLite3Table randomTable = globalState.getSchema().getRandomTableOrBailout(t -> !t.isView() && !t.isReadOnly()); return insertRow(globalState, randomTable); } - public static Query insertRow(SQLite3GlobalState globalState, SQLite3Table randomTable) { + public static SQLQueryAdapter insertRow(SQLite3GlobalState globalState, SQLite3Table randomTable) { SQLite3InsertGenerator generator = new SQLite3InsertGenerator(globalState, globalState.getRandomly()); String query = generator.insertRow(randomTable); - return new QueryAdapter(query, generator.errors, true); + return new SQLQueryAdapter(query, generator.errors, true); } private String insertRow(SQLite3Table table) { - errors.add("cannot UPDATE generated column"); - errors.add("[SQLITE_CONSTRAINT]"); + SQLite3Errors.addInsertUpdateErrors(errors); errors.add("[SQLITE_FULL]"); - errors.add("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch"); - errors.add("[SQLITE_CONSTRAINT] Abort due to constraint violation (FOREIGN KEY constraint failed)"); // // TODO: also check if the table is really missing (caused by a DROP TABLE) - errors.add("[SQLITE_ERROR] SQL error or missing database (no such table:"); errors.add("ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint"); // trigger - errors.add("no such column"); // trigger errors.add("values were supplied"); // trigger errors.add("Data type mismatch (datatype mismatch)"); // trigger - errors.add("too many levels of trigger recursion"); - errors.add("String or BLOB exceeds size limit"); - - errors.add("A table in the database is locked"); - errors.add("cannot INSERT into generated column"); // TODO: filter out generated columns errors.add("load_extension() prohibited in triggers and views"); SQLite3Errors.addInsertNowErrors(errors); @@ -71,7 +60,8 @@ private String insertRow(SQLite3Table table) { sb.append(fromOptions); } boolean defaultValues = false; - sb.append("INTO " + table.getName()); + sb.append("INTO "); + sb.append(table.getName()); List cols = table.getRandomNonEmptyColumnSubset(); if (cols.size() != table.getColumns().size() || Randomly.getBoolean()) { sb.append("("); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java new file mode 100644 index 000000000..c539ad648 --- /dev/null +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3StatTableGenerator.java @@ -0,0 +1,88 @@ +package sqlancer.sqlite3.gen.dml; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; + +public final class SQLite3StatTableGenerator { + + private final SQLite3GlobalState globalState; + + private SQLite3StatTableGenerator(SQLite3GlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(SQLite3GlobalState globalState) { + return new SQLite3StatTableGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + List columns = new ArrayList<>(); + SQLite3Table t = new SQLite3Table("sqlite_stat1", columns, TableKind.MAIN, false, false, false, false); + if (Randomly.getBoolean()) { + return SQLite3DeleteGenerator.deleteContent(globalState, t); + } else { + StringBuilder sb = new StringBuilder(); + sb.append("INSERT OR IGNORE INTO sqlite_stat1"); + String indexName; + try (Statement stat = globalState.getConnection().createStatement()) { + try (ResultSet rs = stat + .executeQuery("SELECT name FROM sqlite_master WHERE type='index' ORDER BY RANDOM() LIMIT 1;")) { + if (rs.isClosed()) { + throw new IgnoreMeException(); + } + indexName = rs.getString("name"); + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + sb.append(" VALUES"); + sb.append("('"); + sb.append(globalState.getSchema().getRandomTable().getName()); + sb.append("', "); + sb.append("'"); + if (Randomly.getBoolean()) { + sb.append(indexName); + } else { + sb.append(globalState.getSchema().getRandomTable().getName()); + } + sb.append("'"); + sb.append(", '"); + for (int i = 0; i < Randomly.smallNumber(); i++) { + if (i != 0) { + sb.append(" "); + } + if (Randomly.getBoolean()) { + sb.append(globalState.getRandomly().getInteger()); + } else { + sb.append(Randomly.smallNumber()); + } + } + if (Randomly.getBoolean()) { + sb.append(" sz="); + sb.append(globalState.getRandomly().getInteger()); + } + if (Randomly.getBoolean()) { + sb.append(" unordered"); + } + if (Randomly.getBoolean()) { + sb.append(" noskipscan"); + } + sb.append("')"); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("no such table", "The database file is locked")); + } + } + +} diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 76e1b4a36..9230f47ba 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -1,44 +1,45 @@ package sqlancer.sqlite3.gen.dml; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3UpdateGenerator { +public final class SQLite3UpdateGenerator extends AbstractUpdateGenerator { - private final StringBuilder sb = new StringBuilder(); - private final Randomly r; - private final List errors = new ArrayList<>(); private final SQLite3GlobalState globalState; + private final Randomly r; + private final SQLite3Table table; - public SQLite3UpdateGenerator(SQLite3GlobalState globalState, Randomly r) { + private SQLite3UpdateGenerator(SQLite3GlobalState globalState, SQLite3Table table) { this.globalState = globalState; - this.r = r; + this.r = globalState.getRandomly(); + this.table = table; + this.canAffectSchema = true; } - public static Query updateRow(SQLite3GlobalState globalState) { + public static SQLQueryAdapter updateRow(SQLite3GlobalState globalState) { SQLite3Table randomTableNoViewOrBailout = globalState.getSchema() - .getRandomTable(t -> !t.isView() && !t.isReadOnly()); + .getRandomTableOrBailout(t -> !t.isView() && !t.isReadOnly()); return updateRow(globalState, randomTableNoViewOrBailout); } - public static Query updateRow(SQLite3GlobalState globalState, SQLite3Table table) { - SQLite3UpdateGenerator generator = new SQLite3UpdateGenerator(globalState, globalState.getRandomly()); - return generator.update(table); + public static SQLQueryAdapter updateRow(SQLite3GlobalState globalState, SQLite3Table table) { + return new SQLite3UpdateGenerator(globalState, table).getStatement(); } - private Query update(SQLite3Table table) { + @Override + public void buildStatement() { + List columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); sb.append("UPDATE "); if (Randomly.getBoolean()) { sb.append("OR IGNORE "); @@ -56,7 +57,6 @@ private Query update(SQLite3Table table) { sb.append(table.getName()); sb.append(" SET "); - List columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); if (Randomly.getBoolean()) { sb.append("("); sb.append(columnsToUpdate.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); @@ -67,26 +67,18 @@ private Query update(SQLite3Table table) { if (i != 0) { sb.append(", "); } - getToUpdateValue(columnsToUpdate, i); + updateValue(columnsToUpdate.get(i)); } sb.append(")"); // row values } else { - for (int i = 0; i < columnsToUpdate.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columnsToUpdate.get(i).getName()); - sb.append(" = "); - getToUpdateValue(columnsToUpdate, i); - } + updateColumns(columnsToUpdate); } if (Randomly.getBoolean()) { - sb.append(" WHERE "); String whereClause = SQLite3Visitor.asString( new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression()); - sb.append(whereClause); + appendWhereClause(whereClause); } // ORDER BY and LIMIT are only supported by enabling a compile-time option @@ -96,32 +88,23 @@ private Query update(SQLite3Table table) { // sb.append(expressions.stream().map(e -> SQLite3Visitor.asString(e)).collect(Collectors.joining(", "))); // } - errors.add("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch"); - errors.add("[SQLITE_CONSTRAINT] Abort due to constraint violation"); + SQLite3Errors.addInsertUpdateErrors(errors); + errors.add("[SQLITE_ERROR] SQL error or missing database (parser stack overflow)"); errors.add( "[SQLITE_ERROR] SQL error or missing database (second argument to likelihood() must be a constant between 0.0 and 1.0)"); - errors.add("[SQLITE_ERROR] SQL error or missing database (no such table:"); // for views errors.add("ORDER BY term out of range"); - errors.add("no such column"); - errors.add("(too many levels of trigger recursion"); - errors.add("String or BLOB exceeds size limit"); - errors.add("cannot UPDATE generated column"); errors.add("unknown function: json_type"); - // TODO not update generated columns? - errors.add("cannot INSERT into generated column"); - errors.add("A table in the database is locked"); // https://www.sqlite.org/src/tktview?name=56a74875be SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addDeleteErrors(errors); - return new QueryAdapter(sb.toString(), errors, true /* column could have an ON UPDATE clause */); - } - private void getToUpdateValue(List columnsToUpdate, int i) { - if (columnsToUpdate.get(i).isIntegerPrimaryKey()) { + @Override + protected void updateValue(SQLite3Column column) { + if (column.isIntegerPrimaryKey()) { sb.append(SQLite3Visitor.asString(SQLite3Constant.createIntConstant(r.getInteger()))); } else { sb.append(SQLite3Visitor.asString(SQLite3ExpressionGenerator.getRandomLiteralValue(globalState))); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java new file mode 100644 index 000000000..532709676 --- /dev/null +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -0,0 +1,981 @@ +package sqlancer.sqlite3.oracle; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.CODDTestBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3Provider; +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Aggregate; +import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.ast.SQLite3Constant.SQLite3TextConstant; +import sqlancer.sqlite3.ast.SQLite3Expression; +import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; +import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; +import sqlancer.sqlite3.ast.SQLite3Select; +import sqlancer.sqlite3.gen.SQLite3Common; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.schema.SQLite3DataType; +import sqlancer.sqlite3.schema.SQLite3Schema; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; + +public class SQLite3CODDTestOracle extends CODDTestBase implements TestOracle { + + private final SQLite3Schema s; + private SQLite3ExpressionGenerator gen; + private Reproducer reproducer; + + private static final String TEMP_TABLE_NAME = "temp_table"; + + private SQLite3Expression foldedExpr; + private SQLite3Expression constantResOfFoldedExpr; + + private List tablesFromOuterContext = new ArrayList<>(); + private List joinsInExpr; + + Map> auxiliaryQueryResult = new HashMap<>(); + Map> selectResult = new HashMap<>(); + + Boolean useSubqueryAsFoldedExpr; + Boolean useCorrelatedSubqueryAsFoldedExpr; + + public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + SQLite3Errors.addExpectedExpressionErrors(errors); + SQLite3Errors.addMatchQueryErrors(errors); + SQLite3Errors.addQueryErrors(errors); + // errors.add("misuse of aggregate"); + // errors.add("misuse of window function"); + // errors.add("second argument to nth_value must be a positive integer"); + // errors.add("no such table"); + // errors.add("no query solution"); + // errors.add("unable to use function MATCH in the requested context"); + // errors.add("[SQLITE_ERROR] SQL error or missing database (unrecognized token:"); + } + + @Override + public void check() throws SQLException { + reproducer = null; + + joinsInExpr = null; + tablesFromOuterContext.clear(); + + useSubqueryAsFoldedExpr = useSubquery(); + useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); + + SQLite3Select auxiliaryQuery = null; + if (useSubqueryAsFoldedExpr) { + if (useCorrelatedSubqueryAsFoldedExpr) { + auxiliaryQuery = genSelectWithCorrelatedSubquery(); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + + auxiliaryQueryResult.putAll(selectResult); + } else { + auxiliaryQuery = genSelectExpression(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); + } + } else { + auxiliaryQuery = genSimpleSelect(); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + + auxiliaryQueryResult.putAll(selectResult); + } + + SQLite3Select originalQuery = null; + + Map> foldedResult = new HashMap<>(); + Map> originalResult = new HashMap<>(); + + // dependent expression + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { + // original query + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(this.foldedExpr); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (auxiliaryQueryResult.isEmpty() + || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).isEmpty()) { + // independent expression + // empty result, put the inner query in (NOT) EXIST + boolean isNegated = !Randomly.getBoolean(); + // original query + SQLite3Exist existExpr = new SQLite3Exist(new SQLite3Select(auxiliaryQuery), isNegated); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(existExpr); + + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + SQLite3Expression equivalentExpr = isNegated ? SQLite3Constant.createTrue() : SQLite3Constant.createFalse(); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (auxiliaryQueryResult.size() == 1 + && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 + && Randomly.getBoolean()) { + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant + // original query + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(auxiliaryQuery); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + SQLite3Expression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]) + .get(0); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability() + && enableInOperator()) { + // one column + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); + SQLite3Table selectedTable = selectedColumn.getColumn().getTable(); + InOperation inOperation = new InOperation(selectedColumn, new SQLite3Select(auxiliaryQuery)); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(inOperation); + + originalQuery = this.genSelectExpression(selectedTable, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + // folded query + // can not use IN VALUES here, because there is no affinity for the right operand of IN when right operand + // is a list + try { + SQLite3Table t = this.createTemporaryTable(auxiliaryQuery, "intable"); + SQLite3TableReference equivalentTable = new SQLite3TableReference(t); + inOperation = new InOperation(selectedColumn, equivalentTable); + specificCondition.updateInnerExpr(inOperation); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable("intable"); + } + } else { + // There is not `ANY` and `ALL` operator in SQLite3 + // Row Subquery + // original query + SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, SQLite3CODDTestOracle.TEMP_TABLE_NAME); + originalQuery = this.genSelectExpression(temporaryTable, null); + SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); + SQLite3WithClause withClause = new SQLite3WithClause(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); + originalQuery.setWithClause(withClause); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + // folded query + if (Randomly.getBoolean() && this.enableCommonTableExpression()) { + // there are too many false positives + // common table expression + // folded query: WITH table AS VALUES () + SQLite3Values values = new SQLite3Values(auxiliaryQueryResult, temporaryTable.getColumns()); + originalQuery.updateWithClauseRight(values); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (Randomly.getBoolean() && this.enableDerivedTable()) { + // derived table + // folded query: SELECT FROM () AS table + originalQuery.setWithClause(null); + SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); + SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); + originalQuery.replaceFromTable(SQLite3CODDTestOracle.TEMP_TABLE_NAME, alias); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (this.enableInsert()) { + // there are too many false positives + // folded query: CREATE the table and INSERT INTO table subquery + try { + this.createTemporaryTable(auxiliaryQuery, SQLite3CODDTestOracle.TEMP_TABLE_NAME); + originalQuery.setWithClause(null); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable(SQLite3CODDTestOracle.TEMP_TABLE_NAME); + } + } else { + throw new IgnoreMeException(); + } + } + if (foldedResult == null || originalResult == null) { + throw new IgnoreMeException(); + } + if (foldedQueryString.equals(originalQueryString)) { + throw new IgnoreMeException(); + } + if (!compareResult(foldedResult, originalResult)) { + reproducer = null; // TODO + state.getState().getLocalState() + .log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError( + auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); + } + } + + private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Expression specificCondition) { + SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); + if (tempTable != null) { + randomTables.addTable(tempTable); + } + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { + for (SQLite3Table t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + randomTables.removeTable(t); + } + } + } + + List columns = randomTables.getColumns(); + if ((!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) + && this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + List joinStatements = new ArrayList<>(); + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { + if (this.joinsInExpr != null) { + joinStatements.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } + } else if (Randomly.getBoolean()) { + joinStatements = genJoinExpression(gen, tables, + Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); + } + List tableRefs = SQLite3Common.getTableRefs(tables, s); + SQLite3Select select = new SQLite3Select(); + select.setFromList(tableRefs); + if (!joinStatements.isEmpty()) { + select.setJoinClauses(joinStatements); + } + + SQLite3Expression randomWhereCondition = gen.generateExpression(); + SQLite3Expression whereCondition = null; + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomWhereCondition, specificCondition, + operator); + } else { + whereCondition = randomWhereCondition; + } + select.setWhereClause(whereCondition); + + if (Randomly.getBoolean()) { + select.setOrderByClauses(genOrderBysExpression(gen, + Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + } + + if (Randomly.getBoolean()) { + List selectedColumns = Randomly.nonEmptySubset(columns); + List selectedAlias = new LinkedList<>(); + for (int i = 0; i < selectedColumns.size(); ++i) { + SQLite3ColumnName originalName = new SQLite3ColumnName(selectedColumns.get(i), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + i), null); + SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); + selectedAlias.add(columnAlias); + } + select.setFetchColumns(selectedAlias); + } else { + SQLite3ColumnName aggr = new SQLite3ColumnName(Randomly.fromList(columns), null); + SQLite3Provider.mustKnowResult = true; + SQLite3Expression originalName = new SQLite3Aggregate(Arrays.asList(aggr), + SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c0"), null); + SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); + select.setFetchColumns(Arrays.asList(columnAlias)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupByClause(columns, specificCondition); + select.setGroupByClause(groupByClause); + if (!groupByClause.isEmpty() && Randomly.getBooleanWithRatherLowProbability()) { + select.setHavingClause(genHavingClause(columns, specificCondition)); + } + } + } + return select; + } + + // For expression test + private SQLite3Select genSimpleSelect() { + SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + tablesFromOuterContext = randomTables.getTables(); + + if (Randomly.getBooleanWithRatherLowProbability()) { + joinsInExpr = genJoinExpression(gen, tables, null, true); + } else { + joinsInExpr = new ArrayList<>(); + } + + List tableRefs = SQLite3Common.getTableRefs(tables, s); + SQLite3Select select = new SQLite3Select(); + select.setFromList(tableRefs); + if (joinsInExpr != null && !joinsInExpr.isEmpty()) { + select.setJoinClauses(joinsInExpr); + } + + SQLite3Expression whereCondition = gen.generateExpression(); + this.foldedExpr = whereCondition; + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (SQLite3Column c : randomTables.getColumns()) { + SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); + SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); + fetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); + SQLite3Alias columnAlias = new SQLite3Alias(whereCondition, aliasName); + fetchColumns.add(columnAlias); + + select.setFetchColumns(fetchColumns); + + Map> queryRes = null; + try { + queryRes = getQueryResult(SQLite3Visitor.asString(select), state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the summary from results + List summary = queryRes.remove("c" + columnIdx); + + List tempColumnList = new ArrayList<>(); + + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + SQLite3Alias cAlias = (SQLite3Alias) fetchColumns.get(i); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOriginalExpression(); + SQLite3Column column = cRef.getColumn(); + String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); + SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); + tempColumnList.add(newColumn); + } + List columnRef = new ArrayList<>(); + for (SQLite3Column c : randomTables.getColumns()) { + columnRef.add(new SQLite3ColumnName(c, null)); + } + if (tempColumnList.size() != queryRes.size()) { + throw new AssertionError(); + } + SQLite3Values values = new SQLite3Values(queryRes, tempColumnList); + this.constantResOfFoldedExpr = new SQLite3ResultMap(values, columnRef, summary, null); + + return select; + } + + private SQLite3Select genSelectWithCorrelatedSubquery() { + SQLite3Tables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); + SQLite3Tables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); + + List innerQueryFromTables = new ArrayList<>(); + for (SQLite3Table t : innerQueryRandomTables.getTables()) { + if (!outerQueryRandomTables.isContained(t)) { + innerQueryFromTables.add(new SQLite3TableReference(t)); + } + } + for (SQLite3Table t : outerQueryRandomTables.getTables()) { + if (innerQueryRandomTables.isContained(t)) { + innerQueryRandomTables.removeTable(t); + + List newColumns = new ArrayList<>(); + for (SQLite3Column c : t.getColumns()) { + SQLite3Column newColumn = new SQLite3Column(c.getName(), c.getType(), false, null, false); + newColumns.add(newColumn); + } + SQLite3Table newTable = new SQLite3Table(t.getName() + "a", newColumns, null, true, false, false, + false); + for (SQLite3Column c : newColumns) { + c.setTable(newTable); + } + innerQueryRandomTables.addTable(newTable); + + SQLite3Alias alias = new SQLite3Alias(new SQLite3TableReference(t), + new SQLite3TableReference(newTable)); + innerQueryFromTables.add(alias); + } + } + + List innerQueryColumns = new ArrayList<>(); + innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); + innerQueryColumns.addAll(outerQueryRandomTables.getColumns()); + gen = new SQLite3ExpressionGenerator(state).setColumns(innerQueryColumns); + + SQLite3Select innerQuery = new SQLite3Select(); + innerQuery.setFromList(innerQueryFromTables); + + SQLite3Expression innerQueryWhereCondition = gen.generateExpression(); + innerQuery.setWhereClause(innerQueryWhereCondition); + + // use aggregate function in fetch column + SQLite3ColumnName innerQueryAggr = new SQLite3ColumnName(Randomly.fromList(innerQueryRandomTables.getColumns()), + null); + SQLite3Provider.mustKnowResult = true; + SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), + SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupByClause(innerQueryColumns, null); + innerQuery.setGroupByClause(groupByClause); + if (!groupByClause.isEmpty() && Randomly.getBooleanWithRatherLowProbability()) { + innerQuery.setHavingClause(genHavingClause(innerQueryColumns, null)); + } + } + + this.foldedExpr = innerQuery; + + // outer query + SQLite3Select outerQuery = new SQLite3Select(); + outerQuery.setFromList(SQLite3Common.getTableRefs(outerQueryRandomTables.getTables(), s)); + tablesFromOuterContext = outerQueryRandomTables.getTables(); + + List outerQueryFetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (SQLite3Column c : outerQueryRandomTables.getColumns()) { + SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); + SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); + outerQueryFetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); + SQLite3Alias columnAlias = new SQLite3Alias(innerQuery, aliasName); + outerQueryFetchColumns.add(columnAlias); + + outerQuery.setFetchColumns(outerQueryFetchColumns); + + originalQueryString = SQLite3Visitor.asString(outerQuery); + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the summary from results + List summary = queryRes.remove("c" + columnIdx); + + List tempColumnList = new ArrayList<>(); + + for (int i = 0; i < outerQueryFetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + SQLite3Alias cAlias = (SQLite3Alias) outerQueryFetchColumns.get(i); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOriginalExpression(); + SQLite3Column column = cRef.getColumn(); + String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); + SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); + tempColumnList.add(newColumn); + } + List columnRef = new ArrayList<>(); + for (SQLite3Column c : outerQueryRandomTables.getColumns()) { + columnRef.add(new SQLite3ColumnName(c, null)); + } + if (tempColumnList.size() != queryRes.size()) { + throw new AssertionError(); + } + SQLite3Values values = new SQLite3Values(queryRes, tempColumnList); + this.constantResOfFoldedExpr = new SQLite3ResultMap(values, columnRef, summary, null); + + return outerQuery; + } + + private List genJoinExpression(SQLite3ExpressionGenerator gen, List tables, + SQLite3Expression specificCondition, boolean joinForExperssion) { + List joinStatements = new ArrayList<>(); + if (!state.getDbmsSpecificOptions().testJoins) { + return joinStatements; + } + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + if (Randomly.getBoolean() && tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1 || joinForExperssion) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + SQLite3Expression randomOnCondition = gen.generateExpression(); + SQLite3Expression onCondition = null; + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + onCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomOnCondition, specificCondition, + operator); + } else { + onCondition = randomOnCondition; + } + + SQLite3Table table = Randomly.fromList(tables); + tables.remove(table); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + onCondition = null; + } + Join j = new SQLite3Expression.Join(table, onCondition, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } + + private List genOrderBysExpression(SQLite3ExpressionGenerator gen, + SQLite3Expression specificCondition) { + List expressions = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + expressions.add( + genOrderingTerm(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + } + return expressions; + } + + private SQLite3Expression genOrderingTerm(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + SQLite3Expression expr = gen.generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + // COLLATE is potentially already generated + if (Randomly.getBoolean()) { + expr = new SQLite3OrderingTerm(expr, Ordering.getRandomValue()); + } + if (state.getDbmsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { + expr = new SQLite3PostfixText(expr, Randomly.fromOptions(" NULLS FIRST", " NULLS LAST"), + null /* expr.getExpectedValue() */) { + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + }; + } + return expr; + } + + private List genGroupByClause(List columns, SQLite3Expression specificCondition) { + errors.add("GROUP BY term out of range"); + if (Randomly.getBoolean()) { + List collect = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(state).setColumns(columns).generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + collect.add(expr); + } + return collect; + } + return Collections.emptyList(); + } + + private SQLite3Expression genHavingClause(List columns, SQLite3Expression specificCondition) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(state).setColumns(columns).generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + return expr; + } + + private Map> getQueryResult(String queryString, SQLite3GlobalState state) + throws SQLException { + Map> result = new LinkedHashMap<>(); + if (options.logEachSelect()) { + logger.writeCurrentNoLineBreak(queryString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + stmt.setQueryTimeout(600); + ResultSet rs = null; + try { + rs = stmt.executeQuery(queryString); + ResultSetMetaData metaData = rs.getMetaData(); + Integer columnCount = metaData.getColumnCount(); + Map idxNameMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + result.put("c" + (i - 1), new ArrayList<>()); + idxNameMap.put(i, "c" + (i - 1)); + } + + int resultRows = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + try { + Object value = rs.getObject(i); + SQLite3Constant constant; + if (rs.wasNull()) { + constant = SQLite3Constant.createNullConstant(); + } else if (value instanceof Integer) { + constant = SQLite3Constant.createIntConstant(Long.valueOf((Integer) value)); + } else if (value instanceof Short) { + constant = SQLite3Constant.createIntConstant(Long.valueOf((Short) value)); + } else if (value instanceof Long) { + constant = SQLite3Constant.createIntConstant((Long) value); + } else if (value instanceof Double) { + constant = SQLite3Constant.createRealConstant((double) value); + } else if (value instanceof Float) { + constant = SQLite3Constant.createRealConstant(((Float) value).doubleValue()); + } else if (value instanceof BigDecimal) { + constant = SQLite3Constant.createRealConstant(((BigDecimal) value).doubleValue()); + } else if (value instanceof byte[]) { + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + } else if (value instanceof Boolean) { + constant = SQLite3Constant.createBoolean((boolean) value); + } else if (value instanceof String) { + constant = SQLite3Constant.createTextConstant((String) value); + } else if (value == null) { + constant = SQLite3Constant.createNullConstant(); + } else { + throw new IgnoreMeException(); + } + List v = result.get(idxNameMap.get(i)); + v.add(constant); + } catch (SQLException e) { + System.out.println(e.getMessage()); + throw new IgnoreMeException(); + } + } + ++resultRows; + if (resultRows > 100) { + throw new IgnoreMeException(); + } + } + Main.nrSuccessfulActions.addAndGet(1); + rs.close(); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + if (rs != null) { + rs.close(); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + return result; + } + + private SQLite3Table genTemporaryTable(SQLite3Select select, String tableName) { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + i; + SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); + databaseColumns.add(column); + } + SQLite3Table table = new SQLite3Table(tableName, databaseColumns, null, false, false, false, false); + for (SQLite3Column c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName) throws SQLException { + String selectString = SQLite3Visitor.asString(select); + Map idxTypeMap = getColumnTypeFromSelect(select); + + Integer columnNumber = idxTypeMap.size(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE " + tableName + " ("); + for (int i = 0; i < columnNumber; ++i) { + String columnTypeName = ""; + if (idxTypeMap.get(i) != null) { + switch (idxTypeMap.get(i)) { + case INT: + case TEXT: + case REAL: + columnTypeName = idxTypeMap.get(i).name(); + break; + case BINARY: + columnTypeName = ""; + break; + default: + columnTypeName = ""; + } + } + sb.append("c" + i + " " + columnTypeName); + if (i < columnNumber - 1) { + sb.append(", "); + } + } + sb.append(");"); + String crateTableString = sb.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(crateTableString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(crateTableString); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + StringBuilder sb2 = new StringBuilder(); + sb2.append("INSERT INTO " + tableName + " " + selectString); + String insertValueString = sb2.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(insertValueString); + } + stmt = null; + try { + stmt = this.con.createStatement(); + try { + Main.nrSuccessfulActions.addAndGet(1); + stmt.execute(insertValueString); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + i; + SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); + databaseColumns.add(column); + } + SQLite3Table table = new SQLite3Table(tableName, databaseColumns, null, false, false, false, false); + for (SQLite3Column c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private void dropTemporaryTable(String tableName) throws SQLException { + String dropString = "DROP TABLE " + tableName + ";"; + if (options.logEachSelect()) { + logger.writeCurrent(dropString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(dropString); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + } + + private boolean compareResult(Map> r1, Map> r2) { + if (r1.size() != r2.size()) { + return false; + } + for (Map.Entry> entry : r1.entrySet()) { + String currentKey = entry.getKey(); + if (!r2.containsKey(currentKey)) { + return false; + } + List v1 = entry.getValue(); + List v2 = r2.get(currentKey); + if (v1.size() != v2.size()) { + return false; + } + List v1Value = new ArrayList<>(v1.stream().map(c -> c.toString()).collect(Collectors.toList())); + List v2Value = new ArrayList<>(v2.stream().map(c -> c.toString()).collect(Collectors.toList())); + Collections.sort(v1Value); + Collections.sort(v2Value); + if (!v1Value.equals(v2Value)) { + return false; + } + } + return true; + } + + private Map getColumnTypeFromSelect(SQLite3Select select) { + List fetchColumns = select.getFetchColumns(); + List newFetchColumns = new ArrayList<>(); + for (SQLite3Expression column : fetchColumns) { + newFetchColumns.add(column); + SQLite3Alias columnAlias = (SQLite3Alias) column; + SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOriginalExpression()); + newFetchColumns.add(typeofColumn); + } + SQLite3Select newSelect = new SQLite3Select(select); + newSelect.setFetchColumns(newFetchColumns); + Map> typeResult = null; + try { + typeResult = getQueryResult(SQLite3Visitor.asString(newSelect), state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + + if (typeResult == null) { + throw new IgnoreMeException(); + } + Map idxTypeMap = new HashMap<>(); + for (int i = 0; i * 2 < typeResult.size(); ++i) { + String columnName = "c" + (i * 2 + 1); + SQLite3Expression t = typeResult.get(columnName).get(0); + SQLite3TextConstant tString = (SQLite3TextConstant) t; + String typeName = tString.asString(); + SQLite3DataType cType = SQLite3DataType.getTypeFromName(typeName); + idxTypeMap.put(i, cType); + } + + return idxTypeMap; + } + + public boolean useSubquery() { + if (this.state.getDbmsSpecificOptions().coddTestModel.isRandom()) { + return Randomly.getBoolean(); + } else if (this.state.getDbmsSpecificOptions().coddTestModel.isExpression()) { + return false; + } else if (this.state.getDbmsSpecificOptions().coddTestModel.isSubquery()) { + return true; + } else { + System.out.printf("Wrong option of --coddtest-model, should be one of: RANDOM, EXPRESSION, SUBQUERY"); + System.exit(1); + return false; + } + } + + public boolean useCorrelatedSubquery() { + return Randomly.getBoolean(); + } + + public boolean enableCommonTableExpression() { + return false; + } + + public boolean enableDerivedTable() { + return true; + } + + public boolean enableInsert() { + return false; + } + + public boolean enableInOperator() { + return false; + } + + @Override + public String getLastQueryString() { + return originalQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index 57190ce08..e98e5f032 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -1,16 +1,13 @@ package sqlancer.sqlite3.oracle; -import java.sql.SQLException; - -import sqlancer.MainOptions; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.TestOracle; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; // tries to trigger a crash -public class SQLite3Fuzzer implements TestOracle { +public class SQLite3Fuzzer implements TestOracle { private final SQLite3GlobalState globalState; @@ -19,24 +16,12 @@ public SQLite3Fuzzer(SQLite3GlobalState globalState) { } @Override - public void check() throws SQLException { + public void check() throws Exception { String s = SQLite3Visitor .asString(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + ";"; - MainOptions options = globalState.getOptions(); try { - if (options.logEachSelect()) { - globalState.getLogger().writeCurrent(s); - } - if (globalState.getDmbsSpecificOptions().printStatements) { - System.out.println(s); - } - if (globalState.getDmbsSpecificOptions().executeQuery) { - globalState.getManager().execute(new QueryAdapter(s)); - if (globalState.getDmbsSpecificOptions().executeStatementsAndPrintSuccessfulOnes) { - System.out.println(s); - } - globalState.getManager().incrementSelectQueryCount(); - } + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); } catch (Error e) { } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java deleted file mode 100644 index 3e0d555ab..000000000 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ /dev/null @@ -1,174 +0,0 @@ -package sqlancer.sqlite3.oracle; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import sqlancer.IgnoreMeException; -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; -import sqlancer.QueryAdapter; -import sqlancer.Randomly; -import sqlancer.StateToReproduce.SQLite3StateToReproduce; -import sqlancer.TestOracle; -import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Visitor; -import sqlancer.sqlite3.ast.SQLite3Aggregate; -import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.ast.SQLite3Expression.Join; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; -import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.gen.SQLite3Common; -import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; -import sqlancer.sqlite3.schema.SQLite3Schema; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; - -public class SQLite3NoRECOracle implements TestOracle { - - private static final int NO_VALID_RESULT = -1; - private final SQLite3Schema s; - private final SQLite3StateToReproduce state; - private final Set errors = new HashSet<>(); - private final StateLogger logger; - private final MainOptions options; - private final SQLite3GlobalState globalState; - private SQLite3ExpressionGenerator gen; - private String firstQueryString; - private String secondQueryString; - - public SQLite3NoRECOracle(SQLite3GlobalState globalState) { - this.s = globalState.getSchema(); - this.state = (SQLite3StateToReproduce) globalState.getState(); - this.logger = globalState.getLogger(); - this.options = globalState.getOptions(); - this.globalState = globalState; - SQLite3Errors.addExpectedExpressionErrors(errors); - SQLite3Errors.addMatchQueryErrors(errors); - SQLite3Errors.addQueryErrors(errors); - errors.add("misuse of aggregate"); - errors.add("misuse of window function"); - errors.add("second argument to nth_value must be a positive integer"); - errors.add("no such table"); - errors.add("no query solution"); - errors.add("unable to use function MATCH in the requested context"); - } - - @Override - public void check() throws SQLException { - SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - gen = new SQLite3ExpressionGenerator(globalState).setColumns(columns); - SQLite3Expression randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List tableRefs = SQLite3Common.getTableRefs(tables, s); - SQLite3Select select = new SQLite3Select(); - select.setFromTables(tableRefs); - select.setJoinClauses(joinStatements); - - int optimizedCount = getOptimizedQuery(select, randomWhereCondition); - int unoptimizedCount = getUnoptimizedQuery(select, randomWhereCondition); - if (optimizedCount == NO_VALID_RESULT || unoptimizedCount == NO_VALID_RESULT) { - throw new IgnoreMeException(); - } - if (optimizedCount != unoptimizedCount) { - state.queryString = firstQueryString + ";\n" + secondQueryString + ";"; - throw new AssertionError(optimizedCount + " " + unoptimizedCount); - } - - } - - private int getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWhereCondition) throws SQLException { - SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, - randomWhereCondition); - SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); - select.setFetchColumns(Arrays.asList(asText)); - select.setWhereClause(null); - secondQueryString = "SELECT SUM(count) FROM (" + SQLite3Visitor.asString(select) + ")"; - if (options.logEachSelect()) { - logger.writeCurrent(secondQueryString); - } - QueryAdapter q = new QueryAdapter(secondQueryString, errors); - return extractCounts(q); - } - - private int getOptimizedQuery(SQLite3Select select, SQLite3Expression randomWhereCondition) throws SQLException { - boolean useAggregate = Randomly.getBoolean(); - if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } - if (useAggregate) { - select.setFetchColumns(Arrays.asList(new SQLite3Aggregate(Collections.emptyList(), - SQLite3Aggregate.SQLite3AggregateFunction.COUNT_ALL))); - } else { - SQLite3ColumnName aggr = new SQLite3ColumnName(SQLite3Column.createDummy("*"), null); - select.setFetchColumns(Arrays.asList(aggr)); - } - select.setWhereClause(randomWhereCondition); - firstQueryString = SQLite3Visitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(firstQueryString); - } - QueryAdapter q = new QueryAdapter(firstQueryString, errors); - return useAggregate ? extractCounts(q) : countRows(q); - } - - private int countRows(QueryAdapter q) { - int count = 0; - try (ResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return NO_VALID_RESULT; - } else { - try { - while (rs.next()) { - count++; - } - } catch (SQLException e) { - count = NO_VALID_RESULT; - } - rs.getStatement().close(); - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw (IgnoreMeException) e; - } - throw new AssertionError(secondQueryString, e); - } - return count; - } - - private int extractCounts(QueryAdapter q) { - int count = 0; - try (ResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return NO_VALID_RESULT; - } else { - try { - while (rs.next()) { - count += rs.getInt(1); - } - } catch (SQLException e) { - count = NO_VALID_RESULT; - } - rs.getStatement().close(); - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw (IgnoreMeException) e; - } - throw new AssertionError(secondQueryString, e); - } - return count; - } - -} diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 3a479ebd6..98a30dc23 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -1,9 +1,6 @@ package sqlancer.sqlite3.oracle; -import java.sql.Connection; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -11,13 +8,14 @@ import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.StateToReproduce.SQLite3StateToReproduce; -import sqlancer.TestOracle; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3ToStringVisitor; +import sqlancer.SQLConnection; +import sqlancer.StateToReproduce.OracleRunReproductionState; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; @@ -28,8 +26,6 @@ import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; @@ -45,111 +41,102 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3PivotedQuerySynthesisOracle implements TestOracle { +public class SQLite3PivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { - private final Connection database; - private final SQLite3Schema s; - private final Randomly r; - private SQLite3StateToReproduce state; - private SQLite3RowValue rw; private List fetchColumns; - private final List errors = new ArrayList<>(); - private List colExpressions; - private final SQLite3GlobalState globalState; + private OracleRunReproductionState localState; - public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) throws SQLException { - this.database = globalState.getConnection(); - this.r = globalState.getRandomly(); - this.globalState = globalState; - s = SQLite3Schema.fromConnection(globalState); + public SQLite3PivotedQuerySynthesisOracle(SQLite3GlobalState globalState) { + super(globalState); } @Override - public void check() throws SQLException { - Query query = getQueryThatContainsAtLeastOneRow(globalState); - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(query.getQueryString()); - } - boolean isContainedIn = isContainedIn(query); - if (!isContainedIn) { - throw new AssertionError(query); - } - } - - public Query getQueryThatContainsAtLeastOneRow(SQLite3GlobalState state) throws SQLException { - SQLite3Select selectStatement = getQuery(state); - SQLite3ToStringVisitor visitor = new SQLite3ToStringVisitor(); - visitor.visit(selectStatement); - String queryString = visitor.get(); - addExpectedErrors(errors); - return new QueryAdapter(queryString, errors); - } - - public static void addExpectedErrors(List errors) { - errors.add("no such index"); - errors.add("no query solution"); - errors.add( - "[SQLITE_ERROR] SQL error or missing database (second argument to likelihood() must be a constant between 0.0 and 1.0)"); - errors.add("[SQLITE_ERROR] SQL error or missing database (integer overflow)"); - errors.add("[SQLITE_ERROR] SQL error or missing database (parser stack overflow)"); - errors.add("second argument to nth_value must be a positive integer"); - errors.add("misuse of aggregate"); - errors.add("GROUP BY term out of range"); + public Query getRectifiedQuery() throws SQLException { + SQLite3Select selectStatement = getQuery(); + SQLite3Errors.addExpectedExpressionErrors(errors); + return new SQLQueryAdapter(SQLite3Visitor.asString(selectStatement), errors); } - public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLException { - this.state = (SQLite3StateToReproduce) globalState.getState(); - if (s.getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - SQLite3Tables randomFromTables = s.getRandomTableNonEmptyTables(); + public SQLite3Select getQuery() throws SQLException { + assert !globalState.getSchema().getDatabaseTables().isEmpty(); + localState = globalState.getState().getLocalState(); + assert localState != null; + SQLite3Tables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); List tables = randomFromTables.getTables(); - globalState.getState().queryTargetedTablesString = randomFromTables.tableNamesAsString(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); SQLite3Select selectStatement = new SQLite3Select(); selectStatement.setSelectType(Randomly.fromOptions(SQLite3Select.SelectType.values())); List columns = randomFromTables.getColumns(); - for (SQLite3Table t : tables) { - if (t.getRowid() != null) { - columns.add(t.getRowid()); - } - } - rw = randomFromTables.getRandomRowValue(database, (SQLite3StateToReproduce) globalState.getState()); - - List joinStatements = new ArrayList<>(); - for (int i = 1; i < tables.size(); i++) { - SQLite3Expression joinClause = generateWhereClauseThatContainsRowValue(columns, rw); - SQLite3Table table = Randomly.fromList(tables); - tables.remove(table); - JoinType options; - options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS, JoinType.OUTER); - if (options == JoinType.OUTER && tables.size() > 2) { - errors.add("ON clause references tables to its right"); - } - Join j = new SQLite3Expression.Join(table, joinClause, options); - joinStatements.add(j); - } - selectStatement.setJoinClauses(joinStatements); - selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, s)); - - // TODO: also implement a wild-card check (*) // filter out row ids from the select because the hinder the reduction process // once a bug is found - List columnsWithoutRowid = columns.stream().filter(c -> !c.getName().matches("rowid")) - .collect(Collectors.toList()); + List columnsWithoutRowid = columns.stream() + .filter(c -> !SQLite3Schema.ROWID_STRINGS.contains(c.getName())).collect(Collectors.toList()); + List joinStatements = getJoinStatements(globalState, tables, columnsWithoutRowid); + selectStatement.setJoinClauses(joinStatements); + selectStatement.setFromList(SQLite3Common.getTableRefs(tables, globalState.getSchema())); + fetchColumns = Randomly.nonEmptySubset(columnsWithoutRowid); - colExpressions = new ArrayList<>(); List allTables = new ArrayList<>(); allTables.addAll(tables); allTables.addAll(joinStatements.stream().map(join -> join.getTable()).collect(Collectors.toList())); - boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows() == 1); + boolean allTablesContainOneRow = allTables.stream().allMatch(t -> t.getNrRows(globalState) == 1); + boolean testAggregateFunctions = allTablesContainOneRow && globalState.getOptions().testAggregateFunctionsPQS(); + pivotRowExpression = getColExpressions(testAggregateFunctions, columnsWithoutRowid); + selectStatement.setFetchColumns(pivotRowExpression); + SQLite3Expression whereClause = generateRectifiedExpression(columnsWithoutRowid, pivotRow, false); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columnsWithoutRowid, pivotRow, + allTablesContainOneRow); + selectStatement.setGroupByClause(groupByClause); + SQLite3Expression limitClause = generateLimit((long) (Math.pow(globalState.getOptions().getMaxNumberInserts(), + joinStatements.size() + randomFromTables.getTables().size()))); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + SQLite3Expression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + /* PQS does not check for ordering, so we can generate any ORDER BY clause */ + List orderBy = new SQLite3ExpressionGenerator(globalState).generateOrderBys(); + selectStatement.setOrderByClauses(orderBy); + if (!groupByClause.isEmpty() && Randomly.getBoolean()) { + selectStatement.setHavingClause(generateRectifiedExpression(columns, pivotRow, true)); + } + return selectStatement; + } + + private List getJoinStatements(SQLite3GlobalState globalState, List tables, + List columns) { + List joinStatements = new SQLite3ExpressionGenerator(globalState).getRandomJoinClauses(tables); + for (Join j : joinStatements) { + if (j.getType() == JoinType.NATURAL) { + /* NATURAL joins have no on clause and cannot be rectified */ + j.setType(JoinType.INNER); + } + // ensure that the join does not exclude the pivot row + j.setOnClause(generateRectifiedExpression(columns, pivotRow, false)); + } + errors.add("ON clause references tables to its right"); + return joinStatements; + } + + private List getColExpressions(boolean testAggregateFunctions, List columns) { + List colExpressions = new ArrayList<>(); + for (SQLite3Column c : fetchColumns) { - SQLite3Expression colName = new SQLite3ColumnName(c, rw.getValues().get(c)); - if (allTablesContainOneRow && Randomly.getBoolean()) { - boolean generateDistinct = Randomly.getBoolean(); + SQLite3Expression colName = new SQLite3ColumnName(c, pivotRow.getValues().get(c)); + if (testAggregateFunctions && Randomly.getBoolean()) { + + /* + * PQS cannot detect omitted or incorrectly-fetched duplicate rows, so we can generate DISTINCT + * statements + */ + boolean generateDistinct = Randomly.getBooleanWithRatherLowProbability(); if (generateDistinct) { colName = new SQLite3Distinct(colName); } + SQLite3AggregateFunction aggFunc = SQLite3AggregateFunction.getRandom(c.getType()); colName = new SQLite3Aggregate(Arrays.asList(colName), aggFunc); if (Randomly.getBoolean() && !generateDistinct) { @@ -159,122 +146,70 @@ public SQLite3Select getQuery(SQLite3GlobalState globalState) throws SQLExceptio } if (Randomly.getBoolean()) { SQLite3Expression randomExpression; - do { - randomExpression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) - .generateExpression(); - } while (randomExpression.getExpectedValue() == null); + randomExpression = new SQLite3ExpressionGenerator(globalState).setColumns(columns) + .generateResultKnownExpression(); colExpressions.add(randomExpression); } else { colExpressions.add(colName); } } - if (Randomly.getBoolean() && allTablesContainOneRow) { - SQLite3WindowFunction windowFunction = SQLite3WindowFunction.getRandom(columnsWithoutRowid, globalState); - SQLite3Expression windowExpr = generateWindowFunction(columnsWithoutRowid, windowFunction, false); + if (testAggregateFunctions) { + SQLite3WindowFunction windowFunction = SQLite3WindowFunction.getRandom(columns, globalState); + SQLite3Expression windowExpr = generateWindowFunction(columns, windowFunction, false); colExpressions.add(windowExpr); } - selectStatement.setFetchColumns(colExpressions); - globalState.getState().queryTargetedColumnsString = fetchColumns.stream().map(c -> c.getFullQualifiedName()) - .collect(Collectors.joining(", ")); - SQLite3Expression whereClause = generateWhereClauseThatContainsRowValue(columns, rw); - selectStatement.setWhereClause(whereClause); - ((SQLite3StateToReproduce) globalState.getState()).whereClause = selectStatement; - List groupByClause = generateGroupByClause(columns, rw, allTablesContainOneRow); - selectStatement.setGroupByClause(groupByClause); - SQLite3Expression limitClause = generateLimit((long) (Math.pow(globalState.getOptions().getMaxNumberInserts(), - joinStatements.size() + randomFromTables.getTables().size()))); - selectStatement.setLimitClause(limitClause); - if (limitClause != null) { - SQLite3Expression offsetClause = generateOffset(); - selectStatement.setOffsetClause(offsetClause); - } - List orderBy = generateOrderBy(columns); - selectStatement.setOrderByExpressions(orderBy); - if (!groupByClause.isEmpty() && Randomly.getBoolean()) { - SQLite3Expression randomExpression = SQLite3Common.getTrueExpression(columns, globalState); - if (Randomly.getBoolean()) { - SQLite3AggregateFunction aggFunc = SQLite3AggregateFunction.getRandom(); - randomExpression = new SQLite3Aggregate(Arrays.asList(randomExpression), aggFunc); + for (SQLite3Expression expr : colExpressions) { + if (expr.getExpectedValue() == null) { + throw new IgnoreMeException(); } - selectStatement.setHavingClause(randomExpression); } - return selectStatement; + return colExpressions; } private SQLite3Expression generateOffset() { if (Randomly.getBoolean()) { - // OFFSET 0 return SQLite3Constant.createIntConstant(0); } else { return null; } } - public static boolean shouldIgnoreException(SQLException e) { - return e.getMessage().contentEquals("[SQLITE_ERROR] SQL error or missing database (integer overflow)") - || e.getMessage().startsWith("[SQLITE_ERROR] SQL error or missing database (parser stack overflow)") - || e.getMessage().startsWith( - "[SQLITE_ERROR] SQL error or missing database (second argument to likelihood() must be a constant between 0.0 and 1.0)") - || e.getMessage().contains("second argument to nth_value must be a positive integer"); - } - - private boolean isContainedIn(Query query) throws SQLException { - Statement createStatement; - createStatement = database.createStatement(); - + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("SELECT "); - addExpectedValues(sb); - StringBuilder sb2 = new StringBuilder(); - addExpectedValues(sb2); - state.values = sb2.toString(); + String checkForContainmentValues = getGeneralizedPivotRowValues(); + sb.append(checkForContainmentValues); + globalState.getState().getLocalState() + .log("-- we expect the following expression to be contained in the result set: " + + checkForContainmentValues); sb.append(" INTERSECT SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions - sb.append(query.getQueryString()); + sb.append(query.getUnterminatedQueryString()); sb.append(")"); String resultingQueryString = sb.toString(); - state.queryString = resultingQueryString; - Query finalQuery = new QueryAdapter(resultingQueryString, query.getExpectedErrors()); - try (ResultSet result = createStatement.executeQuery(finalQuery.getQueryString())) { - boolean isContainedIn = !result.isClosed(); - createStatement.close(); - return isContainedIn; - } catch (SQLException e) { - for (String exp : finalQuery.getExpectedErrors()) { - if (e.getMessage().contains(exp)) { - return true; - } - } - throw e; - } + return new SQLQueryAdapter(resultingQueryString, query.getExpectedErrors()); } - private void addExpectedValues(StringBuilder sb) { - for (int i = 0; i < colExpressions.size(); i++) { + private String getGeneralizedPivotRowValues() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < pivotRowExpression.size(); i++) { if (i != 0) { sb.append(", "); } - SQLite3Constant expectedValue = colExpressions.get(i).getExpectedValue(); - sb.append(SQLite3Visitor.asString(expectedValue)); - } - } - - public List generateOrderBy(List columns) { - List orderBys = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber(); i++) { - SQLite3Expression expr; - expr = new SQLite3ExpressionGenerator(globalState).setColumns(columns).generateExpression(); - Ordering order = Randomly.fromOptions(Ordering.ASC, Ordering.DESC); - orderBys.add(new SQLite3OrderingTerm(expr, order)); - // TODO RANDOM() + SQLite3Constant expectedValue = pivotRowExpression.get(i).getExpectedValue(); + String value = SQLite3Visitor.asString(expectedValue); + if (value.contains("�") || value.contains("\0")) { + // encoding issues || Java does not completely strings with \0 characters + throw new IgnoreMeException(); + } + sb.append(value); } - // TODO collate - errors.add("ORDER BY term out of range"); - return orderBys; + return sb.toString(); } private SQLite3Expression generateLimit(long l) { if (Randomly.getBoolean()) { - return SQLite3Constant.createIntConstant(r.getLong(l, Long.MAX_VALUE)); + return SQLite3Constant.createIntConstant(globalState.getRandomly().getLong(l, Long.MAX_VALUE)); } else { return null; } @@ -307,27 +242,37 @@ private List generateGroupByClause(List column } } - private SQLite3Expression generateWhereClauseThatContainsRowValue(List columns, SQLite3RowValue rw) { - - return generateNewExpression(columns, rw); - - } - - private SQLite3Expression generateNewExpression(List columns, SQLite3RowValue rw) { - do { - SQLite3Expression expr = new SQLite3ExpressionGenerator(globalState).setRowValue(rw).setColumns(columns) - .generateExpression(); - if (expr.getExpectedValue() != null) { - if (expr.getExpectedValue().isNull()) { - return new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); - } - if (SQLite3Cast.isTrue(expr.getExpectedValue()).get()) { - return expr; - } else { - return new SQLite3UnaryOperation(UnaryOperator.NOT, expr); - } - } - } while (true); + /** + * Generates a predicate that is guaranteed to evaluate to true for the given pivot row. PQS uses this + * method to generate predicates used in WHERE and JOIN clauses. See step 4 of the PQS paper. + * + * @param columns + * @param pivotRow + * @param allowAggregates + * + * @return an expression that evaluates to true. + */ + private SQLite3Expression generateRectifiedExpression(List columns, SQLite3RowValue pivotRow, + boolean allowAggregates) { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState).setRowValue(pivotRow) + .setColumns(columns); + if (allowAggregates) { + gen = gen.allowAggregateFunctions(); + } + SQLite3Expression expr = gen.generateResultKnownExpression(); + SQLite3Expression rectifiedPredicate; + if (expr.getExpectedValue().isNull()) { + // the expr evaluates to NULL => rectify to "expr IS NULL" + rectifiedPredicate = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, expr); + } else if (SQLite3Cast.isTrue(expr.getExpectedValue()).get()) { + // the expr evaluates to TRUE => we can directly return it + rectifiedPredicate = expr; + } else { + // the expr evaluates to FALSE 0> rectify to "NOT expr" + rectifiedPredicate = new SQLite3UnaryOperation(UnaryOperator.NOT, expr); + } + rectifiedPredicates.add(rectifiedPredicate); + return rectifiedPredicate; } // @@ -363,10 +308,8 @@ private SQLite3Expression generateWindowFunction(List columns, SQ default: throw new AssertionError(); } - // sb.append(" BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING"); if (Randomly.getBoolean()) { sb.append(" EXCLUDE "); - // "CURRENT ROW", "GROUP" sb.append(Randomly.fromOptions("NO OTHERS", "TIES")); } } @@ -378,7 +321,7 @@ private SQLite3Expression generateWindowFunction(List columns, SQ private void appendFilter(List columns, StringBuilder sb) { sb.append(" FILTER (WHERE "); - sb.append(SQLite3Visitor.asString(generateWhereClauseThatContainsRowValue(columns, rw))); + sb.append(SQLite3Visitor.asString(generateRectifiedExpression(columns, pivotRow, false))); sb.append(")"); } @@ -401,4 +344,9 @@ private enum FrameSpec { BETWEEN, UNBOUNDED_PRECEDING, CURRENT_ROW } + @Override + protected String getExpectedValues(SQLite3Expression expr) { + return SQLite3Visitor.asExpectedValues(expr); + } + } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java index 48c8aa391..f8f53ae3b 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java @@ -4,7 +4,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Select; @@ -37,10 +37,11 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz SQLite3Tables targetTables = s.getRandomTableNonEmptyTables(); List expressions = new ArrayList<>(); SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState) - .setColumns(s.getTables().getColumns()); - SQLite3ExpressionGenerator whereClauseGen = new SQLite3ExpressionGenerator(globalState); + .setColumns(targetTables.getColumns()); + SQLite3ExpressionGenerator whereClauseGen = new SQLite3ExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); SQLite3ExpressionGenerator aggregateGen = new SQLite3ExpressionGenerator(globalState) - .setColumns(s.getTables().getColumns()).allowAggregateFunctions(); + .setColumns(targetTables.getColumns()).allowAggregateFunctions(); // SELECT SQLite3Select select = new SQLite3Select(); @@ -102,7 +103,8 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz select.setFromList(SQLite3Common.getTableRefs(tables, s)); // TODO: no values are referenced from this sub query yet // if (Randomly.getBooleanWithSmallProbability()) { - // select.getFromList().add(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)); + // select.getFromList().add(SQLite3RandomQuerySynthesizer.generate(globalState, + // Randomly.smallNumber() + 1)); // } // WHERE @@ -121,7 +123,7 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { // ORDER BY - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBooleanWithRatherLowProbability()) { // LIMIT diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 6a693d3d1..4d1dd8529 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -1,18 +1,18 @@ package sqlancer.sqlite3.oracle.tlp; -import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; @@ -28,11 +28,12 @@ import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3TLPAggregateOracle implements TestOracle { +public class SQLite3TLPAggregateOracle implements TestOracle { private final SQLite3GlobalState state; - private final List errors = new ArrayList<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private SQLite3ExpressionGenerator gen; + private String generatedQueryString; public SQLite3TLPAggregateOracle(SQLite3GlobalState state) { this.state = state; @@ -53,10 +54,10 @@ public void check() throws SQLException { List from = SQLite3Common.getTableRefs(targetTables.getTables(), s); select.setFromList(from); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } String originalQuery = SQLite3Visitor.asString(select); - + generatedQueryString = originalQuery; SQLite3Expression whereClause = gen.generateExpression(); SQLite3UnaryOperation negatedClause = new SQLite3UnaryOperation(UnaryOperator.NOT, whereClause); SQLite3PostfixUnaryOperation notNullClause = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, @@ -65,7 +66,9 @@ public void check() throws SQLException { SQLite3Select leftSelect = getSelect(aggregate, from, whereClause); SQLite3Select middleSelect = getSelect(aggregate, from, negatedClause); SQLite3Select rightSelect = getSelect(aggregate, from, notNullClause); - String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; + String aggreateMethod = aggregate.getFunc() == SQLite3AggregateFunction.COUNT_ALL + ? SQLite3AggregateFunction.COUNT.toString() : aggregate.getFunc().toString(); + String metamorphicText = "SELECT " + aggreateMethod + "(aggr) FROM ("; metamorphicText += SQLite3Visitor.asString(leftSelect) + " UNION ALL " + SQLite3Visitor.asString(middleSelect) + " UNION ALL " + SQLite3Visitor.asString(rightSelect); metamorphicText += ")"; @@ -74,8 +77,8 @@ public void check() throws SQLException { // state.getState().queryString = "--" + finalText; String firstResult; String secondResult; - QueryAdapter q = new QueryAdapter(originalQuery, errors); - try (ResultSet result = q.executeAndGet(state)) { + SQLQueryAdapter q = new SQLQueryAdapter(originalQuery, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } @@ -85,8 +88,8 @@ public void check() throws SQLException { throw new IgnoreMeException(); } - QueryAdapter q2 = new QueryAdapter(metamorphicText, errors); - try (ResultSet result = q2.executeAndGet(state)) { + SQLQueryAdapter q2 = new SQLQueryAdapter(metamorphicText, errors); + try (SQLancerResultSet result = q2.executeAndGet(state)) { if (result == null) { throw new IgnoreMeException(); } @@ -95,8 +98,8 @@ public void check() throws SQLException { // TODO throw new IgnoreMeException(); } - state.getState().queryString = "--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " - + secondResult; + state.getState().getLocalState() + .log("--" + originalQuery + "\n--" + metamorphicText + "\n-- " + firstResult + "\n-- " + secondResult); if ((firstResult == null && secondResult != null || firstResult != null && !firstResult.contentEquals(secondResult)) && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { @@ -117,9 +120,14 @@ private SQLite3Select getSelect(SQLite3Aggregate aggregate, List errors = new HashSet<>(); +public class SQLite3TLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { SQLite3Schema s; SQLite3Tables targetTables; SQLite3ExpressionGenerator gen; SQLite3Select select; - SQLite3Expression predicate; - SQLite3Expression negatedPredicate; - SQLite3Expression isNullPredicate; public SQLite3TLPBase(SQLite3GlobalState state) { - this.state = state; + super(state); SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addQueryErrors(errors); } @@ -50,17 +41,15 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new SQLite3ExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new SQLite3Select(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); List joinStatements = gen.getRandomJoinClauses(tables); List tableRefs = SQLite3Common.getTableRefs(tables, s); select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); - select.setFromTables(tableRefs); + select.setFromList(tableRefs); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new SQLite3UnaryOperation(UnaryOperator.NOT, predicate); - isNullPredicate = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, predicate); } List generateFetchColumns() { @@ -74,8 +63,9 @@ List generateFetchColumns() { return columns; } - SQLite3Expression generatePredicate() { - return gen.generateExpression(); + @Override + protected ExpressionGenerator getGen() { + return gen; } } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java index 41a2d0dc7..80c55c3e2 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java @@ -5,12 +5,14 @@ import java.util.List; import sqlancer.ComparatorHelper; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Select.SelectType; public class SQLite3TLPDistinctOracle extends SQLite3TLPBase { + private String generatedQueryString; + public SQLite3TLPDistinctOracle(SQLite3GlobalState state) { super(state); } @@ -21,7 +23,7 @@ public void check() throws SQLException { select.setSelectType(SelectType.DISTINCT); select.setWhereClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -37,4 +39,9 @@ public void check() throws SQLException { state); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java index 677bc73ba..3b39ef4c8 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java @@ -7,13 +7,15 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; public class SQLite3TLPGroupByOracle extends SQLite3TLPBase { + private String generatedQueryString; + public SQLite3TLPGroupByOracle(SQLite3GlobalState state) { super(state); } @@ -24,7 +26,7 @@ public void check() throws SQLException { select.setGroupByClause(select.getFetchColumns()); select.setWhereClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -42,10 +44,13 @@ public void check() throws SQLException { @Override List generateFetchColumns() { - List columns = new ArrayList<>(); - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new SQLite3ColumnName(c, null)) + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new SQLite3ColumnName(c, null)) .collect(Collectors.toList()); - return columns; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; } } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 14a3a37b2..248d4db5f 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -3,15 +3,15 @@ import java.sql.SQLException; import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.Join; @@ -29,10 +29,11 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3TLPHavingOracle implements TestOracle { +public class SQLite3TLPHavingOracle implements TestOracle { private final SQLite3GlobalState state; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); + private String generatedQueryString; public SQLite3TLPHavingOracle(SQLite3GlobalState state) { this.state = state; @@ -56,12 +57,12 @@ public void check() throws SQLException { List from = SQLite3Common.getTableRefs(tables, state.getSchema()); select.setJoinClauses(joinStatements); select.setSelectType(SelectType.ALL); - select.setFromTables(from); + select.setFromList(from); // TODO order by? select.setGroupByClause(groupByColumns); select.setHavingClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); SQLite3Expression predicate = gen.getHavingClause(); @@ -84,4 +85,9 @@ public void check() throws SQLException { throw new AssertionError(originalQueryString + ";\n" + combinedString + ";"); } } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java deleted file mode 100644 index 91090e1db..000000000 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ /dev/null @@ -1,43 +0,0 @@ -package sqlancer.sqlite3.oracle.tlp; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Visitor; - -public class SQLite3TLPWhereOracle extends SQLite3TLPBase { - - public SQLite3TLPWhereOracle(SQLite3GlobalState state) { - super(state); - } - - @Override - public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = SQLite3Visitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithSmallProbability(); - if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = SQLite3Visitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = SQLite3Visitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = SQLite3Visitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - -} diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index 87d91f451..8a343fed2 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -1,6 +1,23 @@ package sqlancer.sqlite3.schema; +import sqlancer.IgnoreMeException; + public enum SQLite3DataType { NULL, INT, TEXT, REAL, NONE, BINARY; + public static SQLite3DataType getTypeFromName(String name) { + if (name.equals("integer")) { + return INT; + } else if (name.equals("real")) { + return REAL; + } else if (name.equals("text")) { + return TEXT; + } else if (name.equals("blob")) { + return NONE; + } else if (name.equals("null")) { + return NULL; + } else { + throw new IgnoreMeException(); + } + } } diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 2b9a6d879..fc97929d3 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -1,6 +1,5 @@ package sqlancer.sqlite3.schema; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -10,27 +9,30 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.function.Predicate; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.StateToReproduce.SQLite3StateToReproduce; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.TableIndex; -import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3Provider.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3ToStringVisitor; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Constant; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table.TableKind; -public class SQLite3Schema { +public class SQLite3Schema extends AbstractSchema { - private final List databaseTables; + /** + * All possible aliases for the rowid column. + */ + public static final List ROWID_STRINGS = Collections + .unmodifiableList(Arrays.asList("rowid", "_rowid_", "oid")); private final List indexNames; public List getIndexNames() { @@ -45,14 +47,6 @@ public String getRandomIndexOrBailout() { } } - public SQLite3Table getRandomTableOrBailout() { - if (databaseTables.isEmpty()) { - throw new IgnoreMeException(); - } else { - return Randomly.fromList(getDatabaseTables()); - } - } - public static class SQLite3Column extends AbstractTableColumn { private final boolean isInteger; // "INTEGER" type, not "INT" @@ -84,6 +78,7 @@ public SQLite3Column(String rowId, SQLite3DataType columnType, boolean isInteger this.generated = generated; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -96,6 +91,8 @@ public boolean isOnlyPrimaryKey() { /** * If a table has a single column primary key and the declared type of that column is "INTEGER" and the table is * not a WITHOUT ROWID table, then the column is known as an INTEGER PRIMARY KEY. + * + * @return whether the column is an INTEGER PRIMARY KEY */ public boolean isIntegerPrimaryKey() { return isInteger && isOnlyPrimaryKey() && !getTable().hasWithoutRowid(); @@ -119,69 +116,47 @@ public static SQLite3Constant getConstant(ResultSet randomRowValues, int columnI throws SQLException, AssertionError { Object value; SQLite3Constant constant; - if (randomRowValues.getString(columnIndex) == null) { - value = null; - constant = SQLite3Constant.createNullConstant(); - } else { - switch (valueType) { - case INT: - value = randomRowValues.getLong(columnIndex); - constant = SQLite3Constant.createIntConstant((long) value); - break; - case REAL: - value = randomRowValues.getDouble(columnIndex); - constant = SQLite3Constant.createRealConstant((double) value); - break; - case TEXT: - case NONE: - value = randomRowValues.getString(columnIndex); - constant = SQLite3Constant.createTextConstant((String) value); - break; - case BINARY: - value = randomRowValues.getBytes(columnIndex); - constant = SQLite3Constant.createBinaryConstant((byte[]) value); - break; - default: - throw new AssertionError(valueType); + switch (valueType) { + case INT: + value = randomRowValues.getLong(columnIndex); + constant = SQLite3Constant.createIntConstant((long) value); + break; + case REAL: + value = randomRowValues.getDouble(columnIndex); + if (!Double.isFinite((double) value)) { + // TODO: the JDBC driver seems to sometimes return infinity for NULL values + throw new IgnoreMeException(); } + constant = SQLite3Constant.createRealConstant((double) value); + break; + case TEXT: + case NONE: + value = randomRowValues.getString(columnIndex); + constant = SQLite3Constant.createTextConstant((String) value); + break; + case BINARY: + value = randomRowValues.getBytes(columnIndex); + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + if (((byte[]) value).length == 0) { + // TODO: the JDBC driver seems to sometimes return a zero-length array for NULL values + throw new IgnoreMeException(); + } + break; + case NULL: + return SQLite3Constant.createNullConstant(); + default: + throw new AssertionError(valueType); } return constant; } - public static class SQLite3Tables { - private final List tables; - private final List columns; + public static class SQLite3Tables extends AbstractTables { public SQLite3Tables(List tables) { - this.tables = tables; - columns = new ArrayList<>(); - for (SQLite3Table t : tables) { - columns.addAll(t.getColumns()); - } + super(tables); } - public String tableNamesAsString() { - return tables.stream().map(t -> t.getName()).collect(Collectors.joining(", ")); - } - - public List getTables() { - return tables; - } - - public List getColumns() { - return columns; - } - - public String columnNamesAsString() { - return getColumns().stream().map(t -> t.getTable().getName() + "." + t.getName()) - .collect(Collectors.joining(", ")); - } - - public String columnNamesAsString(Function function) { - return getColumns().stream().map(function).collect(Collectors.joining(", ")); - } - - public SQLite3RowValue getRandomRowValue(Connection con, SQLite3StateToReproduce state) throws SQLException { + public SQLite3RowValue getRandomRowValue(SQLConnection con) throws SQLException { String randomRow = String.format("SELECT %s, %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + c.getName() + ")"), @@ -195,7 +170,8 @@ public SQLite3RowValue getRandomRowValue(Connection con, SQLite3StateToReproduce throw new IgnoreMeException(); } if (!randomRowValues.next()) { - throw new AssertionError("could not find random row! " + randomRow + "\n" + state); + throw new IgnoreMeException(); + // throw new AssertionError("could not find random row! " + randomRow); } for (int i = 0; i < getColumns().size(); i++) { SQLite3Column column = getColumns().get(i); @@ -207,7 +183,6 @@ public SQLite3RowValue getRandomRowValue(Connection con, SQLite3StateToReproduce values.put(column, constant); } assert !randomRowValues.next(); - state.randomRowValues = values; return new SQLite3RowValue(this, values); } @@ -215,7 +190,7 @@ public SQLite3RowValue getRandomRowValue(Connection con, SQLite3StateToReproduce } - public static class SQLite3Table extends AbstractTable { + public static class SQLite3Table extends AbstractRelationalTable { // TODO: why does the SQLite implementation have no table indexes? public enum TableKind { @@ -225,18 +200,16 @@ public enum TableKind { private final TableKind tableType; private SQLite3Column rowid; private final boolean withoutRowid; - private final int nrRows; private final boolean isVirtual; private final boolean isReadOnly; public SQLite3Table(String tableName, List columns, TableKind tableType, boolean withoutRowid, - int nrRows, boolean isView, boolean isVirtual, boolean isReadOnly) { + boolean isView, boolean isVirtual, boolean isReadOnly) { super(tableName, columns, Collections.emptyList(), isView); this.tableType = tableType; this.withoutRowid = withoutRowid; this.isVirtual = isVirtual; this.isReadOnly = isReadOnly; - this.nrRows = nrRows; } public boolean hasWithoutRowid() { @@ -263,10 +236,6 @@ public boolean isSystemTable() { return getName().startsWith("sqlit"); } - public int getNrRows() { - return nrRows; - } - public boolean isTemp() { return tableType == TableKind.TEMP; } @@ -277,97 +246,33 @@ public boolean isReadOnly() { } - public static class SQLite3RowValue { - private final SQLite3Tables tables; - private final Map values; + public static class SQLite3RowValue extends AbstractRowValue { SQLite3RowValue(SQLite3Tables tables, Map values) { - this.tables = tables; - this.values = values; - } - - public SQLite3Tables getTable() { - return tables; - } - - public Map getValues() { - return values; - } - - @Override - public String toString() { - StringBuffer sb = new StringBuffer(); - int i = 0; - for (SQLite3Column c : tables.getColumns()) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(values.get(c)); - } - return sb.toString(); - } - - public String getRowValuesAsString() { - List columnsToCheck = tables.getColumns(); - return getRowValuesAsString(columnsToCheck); - } - - public String getRowValuesAsString(List columnsToCheck) { - StringBuilder sb = new StringBuilder(); - Map expectedValues = getValues(); - for (int i = 0; i < columnsToCheck.size(); i++) { - if (i != 0) { - sb.append(", "); - } - SQLite3Constant expectedColumnValue = expectedValues.get(columnsToCheck.get(i)); - SQLite3ToStringVisitor visitor = new SQLite3ToStringVisitor(); - visitor.visit(expectedColumnValue); - sb.append(visitor.get()); - } - return sb.toString(); + super(tables, values); } } public SQLite3Schema(List databaseTables, List indexNames) { + super(databaseTables); this.indexNames = indexNames; - this.databaseTables = Collections.unmodifiableList(databaseTables); } @Override public String toString() { StringBuffer sb = new StringBuffer(); for (SQLite3Table t : getDatabaseTables()) { - sb.append(t + "\n"); + sb.append(t); + sb.append("\n"); } return sb.toString(); } - public static int getNrRows(SQLite3GlobalState globalState, String table) throws SQLException { - String string = "SELECT COUNT(*) FROM " + table; - List errors = new ArrayList<>(); - errors.add("ORDER BY term out of range"); - errors.addAll(Arrays.asList("second argument to nth_value must be a positive integer", - "ON clause references tables to its right", "no such table", "no query solution", "no such index", - "GROUP BY term", "is circularly defined", "misuse of aggregate", "no such column", - "misuse of window function")); - SQLite3Errors.addExpectedExpressionErrors(errors); - QueryAdapter q = new QueryAdapter(string, errors); - try (ResultSet query = q.executeAndGet(globalState)) { - if (query == null) { - throw new IgnoreMeException(); - } - query.next(); - int int1 = query.getInt(1); - query.getStatement().close(); - return int1; - } - } - public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throws SQLException { List databaseTables = new ArrayList<>(); List indexNames = new ArrayList<>(); - Connection con = globalState.getConnection(); + SQLConnection con = globalState.getConnection(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT name, type as category, sql FROM sqlite_master UNION " @@ -386,7 +291,6 @@ public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throw || tableName.endsWith("_config") || tableName.endsWith("_segdir") || tableName.endsWith("_stat") || tableName.endsWith("_segments") || tableName.contains("_")) { - isReadOnly = true; continue; // TODO } else if (sqlString.contains("using dbstat")) { isReadOnly = true; @@ -401,18 +305,11 @@ public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throw boolean isDbStatsTable = sqlString.contains("using dbstat"); List databaseColumns = getTableColumns(con, tableName, sqlString, isView, isDbStatsTable); - int nrRows; - try { - // FIXME - nrRows = getNrRows(globalState, tableName); - } catch (IgnoreMeException e) { - nrRows = 0; - } SQLite3Table t = new SQLite3Table(tableName, databaseColumns, tableType.contentEquals("temp_table") ? TableKind.TEMP : TableKind.MAIN, withoutRowid, - nrRows, isView, isVirtual, isReadOnly); + isView, isVirtual, isReadOnly); if (isRowIdTable(withoutRowid, isView, isVirtual)) { - String rowId = Randomly.fromOptions("rowid", "_rowid_", "oid"); + String rowId = Randomly.fromList(ROWID_STRINGS); SQLite3Column rowid = new SQLite3Column(rowId, SQLite3DataType.INT, true, null, true); t.addRowid(rowid); rowid.setTable(t); @@ -434,6 +331,10 @@ public static SQLite3Schema fromConnection(SQLite3GlobalState globalState) throw } indexNames.add(name); } + } catch (SQLException e) { + if (!e.getMessage().contains("The database file is locked")) { + throw new AssertionError(e); + } } } @@ -445,7 +346,7 @@ private static boolean isRowIdTable(boolean withoutRowid, boolean isView, boolea return !isView && !isVirtual && !withoutRowid; } - private static List getTableColumns(Connection con, String tableName, String sql, boolean isView, + private static List getTableColumns(SQLConnection con, String tableName, String sql, boolean isView, boolean isDbStatsTable) throws SQLException { List databaseColumns = new ArrayList<>(); try (Statement s2 = con.createStatement()) { @@ -477,7 +378,8 @@ private static List getTableColumns(Connection con, String tableN columnTypeString.contentEquals("INTEGER"), isPrimaryKey, collate)); } } - } catch (Exception e) { + } catch (SQLException e) { + } if (databaseColumns.isEmpty()) { // only generated columns @@ -538,80 +440,55 @@ public static SQLite3DataType getColumnType(String columnTypeString) { return columnType; } - public SQLite3Table getRandomTable() { - return Randomly.fromList(getDatabaseTables()); - } - - public SQLite3Table getRandomTable(Predicate predicate) { - List collect = databaseTables.stream().filter(predicate).collect(Collectors.toList()); - if (collect.isEmpty()) { - throw new IgnoreMeException(); - } - return Randomly.fromList(collect); - } - - public List getTables(Predicate predicate) { - return databaseTables.stream().filter(predicate).collect(Collectors.toList()); - } - - public SQLite3Table getRandomTableOrBailout(Predicate predicate) { - List tables = databaseTables.stream().filter(predicate).collect(Collectors.toList()); - if (tables.isEmpty()) { - throw new IgnoreMeException(); - } else { - return Randomly.fromList(tables); - } - } - public SQLite3Table getRandomVirtualTable() { return getRandomTable(p -> p.isVirtual); } - public List getDatabaseTables() { - return databaseTables; - } - public SQLite3Tables getTables() { - return new SQLite3Tables(databaseTables); + return new SQLite3Tables(getDatabaseTables()); } public SQLite3Tables getRandomTableNonEmptyTables() { - if (databaseTables.isEmpty()) { - throw new IgnoreMeException(); - } - return new SQLite3Tables(Randomly.nonEmptySubset(databaseTables)); - } - - public SQLite3Table getRandomTableNoViewOrBailout() { - List databaseTablesWithoutViews = getDatabaseTablesWithoutViews(); - if (databaseTablesWithoutViews.isEmpty()) { + if (getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } - return Randomly.fromList(databaseTablesWithoutViews); + return new SQLite3Tables(Randomly.nonEmptySubset(getDatabaseTables())); } public SQLite3Table getRandomTableNoViewNoVirtualTable() { return Randomly.fromList(getDatabaseTablesWithoutViewsWithoutVirtualTables()); } - public List getDatabaseTablesWithoutViews() { - return databaseTables.stream().filter(t -> !t.isView()).collect(Collectors.toList()); + public List getDatabaseTablesWithoutViewsWithoutVirtualTables() { + return getDatabaseTables().stream().filter(t -> !t.isView() && !t.isVirtual).collect(Collectors.toList()); } - public List getViews() { - return databaseTables.stream().filter(t -> t.isView()).collect(Collectors.toList()); + public String getFreeVirtualTableName() { + int i = 0; + if (Randomly.getBooleanWithRatherLowProbability()) { + i = (int) Randomly.getNotCachedInteger(0, 100); + } + do { + String tableName = String.format("vt%d", i++); + if (getDatabaseTables().stream().noneMatch(t -> t.getName().equalsIgnoreCase(tableName))) { + return tableName; + } + } while (true); + } - public SQLite3Table getRandomViewOrBailout() { - if (getViews().isEmpty()) { - throw new IgnoreMeException(); - } else { - return Randomly.fromList(getViews()); + public String getFreeRtreeTableName() { + int i = 0; + if (Randomly.getBooleanWithRatherLowProbability()) { + i = (int) Randomly.getNotCachedInteger(0, 100); } - } + do { + String tableName = String.format("rt%d", i++); + if (getDatabaseTables().stream().noneMatch(t -> t.getName().equalsIgnoreCase(tableName))) { + return tableName; + } + } while (true); - public List getDatabaseTablesWithoutViewsWithoutVirtualTables() { - return databaseTables.stream().filter(t -> !t.isView() && !t.isVirtual).collect(Collectors.toList()); } } diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index da647fd98..3849a3c8c 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -2,45 +2,41 @@ // do not make the fields final to avoid warnings public final class TiDBBugs { + // https://github.com/pingcap/tidb/issues/35677 + public static boolean bug35677 = true; - // https://github.com/pingcap/tidb/issues/15987 - public static boolean bug15987 = true; + // https://github.com/pingcap/tidb/issues/35522 + public static boolean bug35522 = true; - // // https://github.com/pingcap/tidb/issues/15988 - public static boolean bug15988 = true; + // https://github.com/pingcap/tidb/issues/35652 + public static boolean bug35652 = true; - // https://github.com/pingcap/tidb/issues/16028 - public static boolean bug16028 = true; + // https://github.com/pingcap/tidb/issues/38295 + public static boolean bug38295 = true; - // https://github.com/pingcap/tidb/issues/16020 - public static boolean bug16020 = true; + // https://github.com/pingcap/tidb/issues/38319 + public static boolean bug38319 = true; - // https://github.com/pingcap/tidb/issues/15990 - public static boolean bug15990 = true; + // https://github.com/pingcap/tidb/issues/44747 + public static boolean bug44747 = true; - // https://github.com/pingcap/tidb/issues/15844 - public static boolean bug15844 = true; + // https://github.com/pingcap/tidb/issues/46556 + public static boolean bug46556 = true; - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/10 - public static boolean bug10 = true; + // https://github.com/pingcap/tidb/issues/46591 + public static boolean bug46591 = true; - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/14 - public static boolean bug14 = true; + // https://github.com/pingcap/tidb/issues/46598 + public static boolean bug46598 = true; - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/15 - public static boolean bug15 = true; + // https://github.com/pingcap/tidb/issues/47346 + public static boolean bug47346 = true; - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/16 - public static boolean bug16 = true; + // https://github.com/pingcap/tidb/issues/47348 + public static boolean bug47348 = true; - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/19 - public static boolean bug19 = true; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/48 - public static boolean bug48 = true; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/50 - public static boolean bug50 = true; + // https://github.com/pingcap/tidb/issues/51525 + public static boolean bug51525 = true; private TiDBBugs() { } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 1c07fb7e3..528acd0bb 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -1,18 +1,24 @@ package sqlancer.tidb; -import java.util.Set; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; public final class TiDBErrors { private TiDBErrors() { } - public static void addExpressionErrors(Set errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("DECIMAL value is out of range"); errors.add("error parsing regexp"); errors.add("BIGINT UNSIGNED value is out of range"); errors.add("Data truncation: Truncated incorrect time value"); errors.add("Data truncation: Incorrect time value"); + errors.add("Data truncation: Incorrect datetime value"); errors.add("overflows double"); errors.add("overflows bigint"); errors.add("strconv.ParseFloat: parsing"); @@ -21,37 +27,67 @@ public static void addExpressionErrors(Set errors) { // functions errors.add("BIGINT value is out of range"); errors.add("doesn't have a default value"); // default - - // known issue: https://github.com/pingcap/tidb/issues/14819 - errors.add("Wrong plan type for dataReaderBuilder"); - + errors.add("is not valid for CHARACTER SET"); errors.add("DOUBLE value is out of range"); + errors.add("Result of space() was larger than max_allowed_packet"); - errors.add("index out of range"); // https://github.com/pingcap/tidb/issues/15810 - errors.add("baseBuiltinFunc.evalString() should never be called, please contact the TiDB team for help"); // https://github.com/pingcap/tidb/issues/15847 - errors.add("unsupport column type for encode 6"); // https://github.com/pingcap/tidb/issues/15850 - - errors.add("Data truncation: %s value is out of range in '%s'"); + errors.add("Data truncat"); errors.add("Truncated incorrect FLOAT value"); errors.add("Bad Number"); + errors.add("strconv.Atoi: parsing"); + errors.add("expected integer"); + errors.add("Duplicate entry"); // regex errors.add("error parsing regexp"); + errors.add("from regexp"); + errors.add("Empty pattern is invalid"); + errors.add("Invalid regexp pattern"); - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/57 - errors.add("For input string: \"+Inf\""); + // To avoid bugs + errors.add("Unknown column"); // https://github.com/pingcap/tidb/issues/35522 + errors.add("Can\'t find column"); // https://github.com/pingcap/tidb/issues/35527 + errors.add("Cannot convert"); // https://github.com/pingcap/tidb/issues/35652 - errors.add("inconsistent index"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/58 + if (TiDBBugs.bug35677) { + errors.add("for function inet_aton"); + } + if (TiDBBugs.bug35522) { + errors.add("ERROR 1054 (42S22)"); + } + if (TiDBBugs.bug35652) { + errors.add("from binary to utf8"); + } + if (TiDBBugs.bug38295) { + errors.add("assertion failed"); + } + if (TiDBBugs.bug44747) { + errors.add("index out of range"); + } + + return errors; + } - errors.add("Illegal mix of collations"); + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); } - public static void addExpressionHavingErrors(Set errors) { + public static List getExpressionHavingErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not in GROUP BY clause and contains nonaggregated column"); errors.add("Unknown column"); + + return errors; } - public static void addInsertErrors(Set errors) { + public static void addExpressionHavingErrors(ExpectedErrors errors) { + errors.addAll(getExpressionHavingErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Duplicate entry"); errors.add("cannot be null"); errors.add("doesn't have a default value"); @@ -69,11 +105,18 @@ public static void addInsertErrors(Set errors) { errors.add("Incorrect bigint value"); errors.add("Incorrect decimal value"); errors.add("error parsing regexp"); + errors.add("is not valid for CHARACTER SET"); + errors.add("for function inet_aton"); + errors.add("'Empty pattern is invalid' from regexp"); + errors.add("Data too long for expression index"); + errors.add("Data too long for column"); + errors.add("Data Too Long"); + + return errors; + } - if (true) { - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/54 - errors.add("Miss column"); - } + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 271a86955..8eaca35f0 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -3,13 +3,20 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.gen.UntypedExpressionGenerator; +import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; import sqlancer.tidb.TiDBSchema.TiDBDataType; +import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.ast.TiDBAggregate; import sqlancer.tidb.ast.TiDBAggregate.TiDBAggregateFunction; import sqlancer.tidb.ast.TiDBBinaryBitOperation; @@ -20,37 +27,160 @@ import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; import sqlancer.tidb.ast.TiDBCase; import sqlancer.tidb.ast.TiDBCastOperation; -import sqlancer.tidb.ast.TiDBCollate; import sqlancer.tidb.ast.TiDBColumnReference; import sqlancer.tidb.ast.TiDBConstant; import sqlancer.tidb.ast.TiDBExpression; import sqlancer.tidb.ast.TiDBFunctionCall; import sqlancer.tidb.ast.TiDBFunctionCall.TiDBFunction; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBJoin.JoinType; import sqlancer.tidb.ast.TiDBOrderingTerm; import sqlancer.tidb.ast.TiDBRegexOperation; import sqlancer.tidb.ast.TiDBRegexOperation.TiDBRegexOperator; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation.TiDBUnaryPostfixOperator; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; -public class TiDBExpressionGenerator extends UntypedExpressionGenerator { - - private final TiDBGlobalState globalState; - - public TiDBExpressionGenerator(TiDBGlobalState globalState) { - this.globalState = globalState; - } +public class TiDBExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator, + CERTGenerator { private enum Gen { UNARY_PREFIX, // UNARY_POSTFIX, // CONSTANT, // COLUMN, // - COMPARISON, REGEX, COLLATE, FUNCTION, BINARY_LOGICAL, BINARY_BIT, CAST, DEFAULT, CASE + COMPARISON, REGEX, FUNCTION, BINARY_LOGICAL, BINARY_BIT, CAST, DEFAULT, CASE // BINARY_ARITHMETIC } + private final TiDBGlobalState globalState; + + private List tables; + + public TiDBExpressionGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public TiDBExpression generateConstant() { + TiDBDataType type = TiDBDataType.getRandom(); + if (Randomly.getBooleanWithRatherLowProbability()) { + return TiDBConstant.createNullConstant(); + } + switch (type) { + case INT: + return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + case BLOB: + case TEXT: + return TiDBConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOL: + return TiDBConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOATING: + return TiDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + case CHAR: + return TiDBConstant.createStringConstant(globalState.getRandomly().getChar()); + case DECIMAL: + case NUMERIC: + return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(); + } + } + + @Override + public List generateOrderBys() { + List expressions = super.generateOrderBys(); + List newExpressions = new ArrayList<>(); + for (TiDBExpression expr : expressions) { + TiDBExpression newExpr = expr; + if (Randomly.getBoolean()) { + newExpr = new TiDBOrderingTerm(expr, Randomly.getBoolean()); + } + newExpressions.add(newExpr); + } + return newExpressions; + } + + @Override + public TiDBExpression negatePredicate(TiDBExpression predicate) { + return new TiDBUnaryPrefixOperation(predicate, TiDBUnaryPrefixOperator.NOT); + } + + @Override + public TiDBExpression isNull(TiDBExpression expr) { + return new TiDBUnaryPostfixOperation(expr, TiDBUnaryPostfixOperator.IS_NULL); + } + + public TiDBExpression generateConstant(TiDBDataType type) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return TiDBConstant.createNullConstant(); + } + switch (type) { + case INT: + return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + case BLOB: + case TEXT: + return TiDBConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOL: + return TiDBConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOATING: + return TiDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + case CHAR: + return TiDBConstant.createStringConstant(globalState.getRandomly().getChar()); + case DECIMAL: + case NUMERIC: + return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(); + } + } + + @Override + public TiDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public TiDBExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public TiDBSelect generateSelect() { + return new TiDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + List joins = TiDBJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> ((TiDBTableReference) t).getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new TiDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBoolean()) { + return List.of(new TiDBColumnReference( + new TiDBColumn("*", new TiDBCompositeDataType(TiDBDataType.INT), false, false, false))); + } + return Randomly.nonEmptySubset(this.columns).stream().map(c -> new TiDBColumnReference(c)) + .collect(Collectors.toList()); + } + @Override protected TiDBExpression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { @@ -64,13 +194,14 @@ protected TiDBExpression generateExpression(int depth) { } switch (Randomly.fromOptions(Gen.values())) { case DEFAULT: - if (TiDBBugs.bug15) { - throw new IgnoreMeException(); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } - return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(generateColumn())); + TiDBColumn column = Randomly.fromList(columns); + if (column.hasDefault()) { + return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(new TiDBColumnReference(column))); + } + throw new IgnoreMeException(); case UNARY_POSTFIX: return new TiDBUnaryPostfixOperation(generateExpression(depth + 1), TiDBUnaryPostfixOperator.getRandom()); case UNARY_PREFIX: @@ -86,39 +217,23 @@ protected TiDBExpression generateExpression(int depth) { case REGEX: return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBRegexOperator.getRandom()); - case COLLATE: - return new TiDBCollate(generateExpression(depth + 1), - Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); case FUNCTION: TiDBFunction func = TiDBFunction.getRandom(); - return new TiDBFunctionCall(func, generateExpressions(depth, func.getNrArgs())); + return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); case BINARY_BIT: return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryBitOperator.getRandom()); case BINARY_LOGICAL: - if (TiDBBugs.bug48) { - throw new IgnoreMeException(); - } return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryLogicalOperator.getRandom()); - // case BINARY_ARITHMETIC: - // return new TiDBBinaryArithmeticOperation(generateExpression(depth + 1), generateExpression(depth + 1), - // TiDBBinaryArithmeticOperator.getRandom()); case CAST: - return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions( - /* - * "BINARY" https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 - */ "CHAR", - /* - * "DATE", "DATETIME", "TIME", https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 - */ "DECIMAL", "SIGNED"/* , "UNSIGNED" https://github.com/pingcap/tidb/issues/16028 */)); + return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 + "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 + "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); case CASE: - if (TiDBBugs.bug19) { - throw new IgnoreMeException(); - } int nr = Randomly.fromOptions(1, 2); - return new TiDBCase(generateExpression(depth + 1), generateExpressions(depth + 1, nr), - generateExpressions(depth + 1, nr), generateExpression(depth + 1)); + return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); default: throw new AssertionError(); } @@ -131,43 +246,134 @@ protected TiDBExpression generateColumn() { } @Override - public TiDBExpression generateConstant() { - TiDBDataType type = TiDBDataType.getRandom(); - if (Randomly.getBooleanWithRatherLowProbability()) { - return TiDBConstant.createNullConstant(); + public String generateExplainQuery(TiDBSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(TiDBSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateJoin); + mutators.add(this::mutateWhere); + if (!TiDBBugs.bug38319) { + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); } - switch (type) { - case INT: - return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); - case BLOB: - case TEXT: - return TiDBConstant.createStringConstant(globalState.getRandomly().getString()); - case BOOL: - return TiDBConstant.createBooleanConstant(Randomly.getBoolean()); - case FLOATING: - return TiDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); - case CHAR: - return TiDBConstant.createStringConstant(globalState.getRandomly().getChar()); - case DECIMAL: - case NUMERIC: - return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); - default: - throw new AssertionError(); + mutators.add(this::mutateAnd); + if (!TiDBBugs.bug51525) { + mutators.add(this::mutateOr); } + mutators.add(this::mutateLimit); + // mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); } - @Override - public List generateOrderBys() { - List expressions = super.generateOrderBys(); - List newExpressions = new ArrayList<>(); - for (TiDBExpression expr : expressions) { - TiDBExpression newExpr = expr; - if (Randomly.getBoolean()) { - newExpr = new TiDBOrderingTerm(expr, Randomly.getBoolean()); + boolean mutateJoin(TiDBSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); + if (join.getJoinType() == JoinType.NATURAL) { + return false; + } + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to + // other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((TiDBTableReference) join.getRightTable()).getTable().getColumns()); + TiDBExpressionGenerator joinGen2 = new TiDBExpressionGenerator(globalState).setColumns(columns); + join.setOnCondition(joinGen2.generateExpression()); + } + + JoinType newJoinType = TiDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + // between LEFT and RIGHT + // join + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); + } else { + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + if (newJoinType == JoinType.CROSS) { + join.setOnCondition(null); + } + return increase; + } + + boolean mutateWhere(TiDBSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression()); + } + return increase; + } + + boolean mutateHaving(TiDBSelect select) { + if (select.getGroupByExpressions().isEmpty()) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; } - newExpressions.add(newExpr); } - return newExpressions; } + boolean mutateAnd(TiDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + TiDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(TiDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + return false; + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + TiDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(TiDBSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(generateConstant(TiDBDataType.INT)); + } + return increase; + } + + private boolean mutateGroupBy(TiDBSelect select) { + boolean increase = !select.getGroupByExpressions().isEmpty(); + if (increase) { + select.clearGroupByExpressions(); + select.clearHavingClause(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } } diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index e60547860..1619832d6 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -1,51 +1,36 @@ package sqlancer.tidb; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; -import sqlancer.CompositeTestOracle; -import sqlancer.MainOptions.DBMSConverter; -import sqlancer.TestOracle; -import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.oracle.TiDBTLPHavingOracle; -import sqlancer.tidb.oracle.TiDBTLPWhereOracle; - -@Parameters -public class TiDBOptions { - - @Parameter(names = "--oracle", converter = DBMSConverter.class) - public List oracle = Arrays.asList(TiDBOracle.QUERY_PARTITIONING); - - public enum TiDBOracle { - HAVING { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBTLPHavingOracle(globalState); - } - }, - WHERE { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBTLPWhereOracle(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); - oracles.add(new TiDBTLPWhereOracle(globalState)); - oracles.add(new TiDBTLPHavingOracle(globalState)); - return new CompositeTestOracle(oracles); - } - }; - - public abstract TestOracle create(TiDBGlobalState globalState) throws SQLException; +import sqlancer.DBMSSpecificOptions; - } +@Parameters(separators = "=", commandDescription = "TiDB (default port: " + TiDBOptions.DEFAULT_PORT + + ", default host: " + TiDBOptions.DEFAULT_HOST + ")") +public class TiDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 4000; + + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/views that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(TiDBOracleFactory.QUERY_PARTITIONING); + @Parameter(names = "--enable-non-prepared-plan-cache") + public boolean nonPreparePlanCache; + + @Parameter(names = { "--tiflash" }, description = "Enable TiFlash") + public boolean tiflash; + + @Override + public List getTestOracleFactory() { + return oracle; + } } diff --git a/src/sqlancer/tidb/TiDBOracleFactory.java b/src/sqlancer/tidb/TiDBOracleFactory.java new file mode 100644 index 000000000..173ff7abd --- /dev/null +++ b/src/sqlancer/tidb/TiDBOracleFactory.java @@ -0,0 +1,77 @@ +package sqlancer.tidb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.tidb.oracle.TiDBDQPOracle; +import sqlancer.tidb.oracle.TiDBTLPHavingOracle; + +public enum TiDBOracleFactory implements OracleFactory { + HAVING { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + return new TiDBTLPHavingOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(2); + return Optional.of((long) Double.parseDouble(content)); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String operation = rs.getString(1).split("_")[0]; // Extract operation names for query plans + return Optional.of(operation); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + DQP { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + return new TiDBDQPOracle(globalState); + } + }; + +} diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 77010cea3..1d117e234 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -5,86 +5,78 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; -import sqlancer.CompositeTestOracle; -import sqlancer.GlobalState; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; -import sqlancer.Main.QueryManager; -import sqlancer.Main.StateLogger; -import sqlancer.ProviderAdapter; -import sqlancer.Query; -import sqlancer.QueryAdapter; -import sqlancer.QueryProvider; +import sqlancer.MainOptions; import sqlancer.Randomly; -import sqlancer.StateToReproduce; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; import sqlancer.tidb.gen.TiDBDeleteGenerator; +import sqlancer.tidb.gen.TiDBDropTableGenerator; +import sqlancer.tidb.gen.TiDBDropViewGenerator; import sqlancer.tidb.gen.TiDBIndexGenerator; import sqlancer.tidb.gen.TiDBInsertGenerator; -import sqlancer.tidb.gen.TiDBRandomQuerySynthesizer; import sqlancer.tidb.gen.TiDBSetGenerator; import sqlancer.tidb.gen.TiDBTableGenerator; import sqlancer.tidb.gen.TiDBUpdateGenerator; import sqlancer.tidb.gen.TiDBViewGenerator; -public class TiDBProvider extends ProviderAdapter { +@AutoService(DatabaseProvider.class) +public class TiDBProvider extends SQLProviderAdapter { public TiDBProvider() { super(TiDBGlobalState.class, TiDBOptions.class); } public enum Action implements AbstractAction { - INSERT(TiDBInsertGenerator::getQuery), // - ANALYZE_TABLE(TiDBAnalyzeTableGenerator::getQuery), // - TRUNCATE((g) -> new QueryAdapter("TRUNCATE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), // - CREATE_INDEX(TiDBIndexGenerator::getQuery), // - DELETE(TiDBDeleteGenerator::getQuery), // - SET(TiDBSetGenerator::getQuery), // - UPDATE(TiDBUpdateGenerator::getQuery), // + CREATE_TABLE(TiDBTableGenerator::createRandomTableStatement), // 0 + CREATE_INDEX(TiDBIndexGenerator::getQuery), // 1 + VIEW_GENERATOR(TiDBViewGenerator::getQuery), // 2 + INSERT(TiDBInsertGenerator::getQuery), // 3 + ALTER_TABLE(TiDBAlterTableGenerator::getQuery), // 4 + TRUNCATE((g) -> new SQLQueryAdapter("TRUNCATE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), // 5 + UPDATE(TiDBUpdateGenerator::getQuery), // 6 + DELETE(TiDBDeleteGenerator::getQuery), // 7 + SET(TiDBSetGenerator::getQuery), // 8 ADMIN_CHECKSUM_TABLE( - (g) -> new QueryAdapter("ADMIN CHECKSUM TABLE " + g.getSchema().getRandomTable().getName())), // - VIEW_GENERATOR(TiDBViewGenerator::getQuery), // - ALTER_TABLE(TiDBAlterTableGenerator::getQuery), // - EXPLAIN((g) -> { - Set errors = new HashSet<>(); - TiDBErrors.addExpressionErrors(errors); - TiDBErrors.addExpressionHavingErrors(errors); - return new QueryAdapter( - "EXPLAIN " + TiDBRandomQuerySynthesizer.generate(g, Randomly.smallNumber() + 1).getQueryString(), - errors); - }); + (g) -> new SQLQueryAdapter("ADMIN CHECKSUM TABLE " + g.getSchema().getRandomTable().getName())), // 9 + ANALYZE_TABLE(TiDBAnalyzeTableGenerator::getQuery), // 10 + DROP_TABLE(TiDBDropTableGenerator::dropTable), // 11 + DROP_VIEW(TiDBDropViewGenerator::dropView); // 12 - private final QueryProvider queryProvider; + private final SQLQueryProvider sqlQueryProvider; - Action(QueryProvider queryProvider) { - this.queryProvider = queryProvider; + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; } @Override - public Query getQuery(TiDBGlobalState state) throws SQLException { - return queryProvider.getQuery(state); + public SQLQueryAdapter getQuery(TiDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); } } - public static class TiDBGlobalState extends GlobalState { + public static class TiDBGlobalState extends SQLGlobalState { - private TiDBSchema schema; - - public void setSchema(TiDBSchema schema) { - this.schema = schema; - } - - public TiDBSchema getSchema() { - return schema; + @Override + protected TiDBSchema readSchema() throws SQLException { + return TiDBSchema.fromConnection(getConnection(), getDatabaseName()); } } @@ -96,7 +88,6 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { case CREATE_INDEX: return r.getInteger(0, 2); case INSERT: - case EXPLAIN: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case TRUNCATE: case DELETE: @@ -110,6 +101,10 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { return r.getInteger(0, 2); case ALTER_TABLE: return r.getInteger(0, 10); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/10 + case CREATE_TABLE: + case DROP_TABLE: + case DROP_VIEW: + return 0; default: throw new AssertionError(a); } @@ -117,101 +112,97 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { } @Override - public void generateAndTestDatabase(TiDBGlobalState globalState) throws SQLException { - QueryManager manager = globalState.getManager(); - Connection con = globalState.getConnection(); - String databaseName = globalState.getDatabaseName(); - globalState.setSchema(TiDBSchema.fromConnection(con, databaseName)); - StateLogger logger = globalState.getLogger(); - StateToReproduce state = globalState.getState(); + public void generateDatabase(TiDBGlobalState globalState) throws Exception { for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { - boolean success = false; + boolean success; do { - Query qt = new TiDBTableGenerator().getQuery(globalState); - success = manager.execute(qt); - logger.writeCurrent(state); - globalState.setSchema(TiDBSchema.fromConnection(con, databaseName)); - try { - logger.getCurrentFileWriter().close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - logger.currentFileWriter = null; + SQLQueryAdapter qt = new TiDBTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); } while (!success); } - globalState.setSchema(TiDBSchema.fromConnection(con, databaseName)); StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), TiDBProvider::mapActions, (q) -> { - if (q.couldAffectSchema()) { - try { - globalState.setSchema(TiDBSchema.fromConnection(con, databaseName)); - } catch (SQLException e) { - if (q.getQueryString().contains("CREATE VIEW") || e.getMessage().contains( - "references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them")) { - throw new IgnoreMeException(); // TODO: drop view instead - } else { - throw new AssertionError(e); - } - } - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } }); - se.executeStatements(); - manager.incrementCreateDatabase(); - List oracles = globalState.getDmbsSpecificOptions().oracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (SQLException e1) { - throw new AssertionError(e1); + try { + se.executeStatements(); + } catch (SQLException e) { + if (e.getMessage().contains( + "references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them")) { + throw new IgnoreMeException(); // TODO: drop view instead + } else { + throw new AssertionError(e); } - }).collect(Collectors.toList()); - CompositeTestOracle oracle = new CompositeTestOracle(oracles); + } - for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { - try { - oracle.check(); - manager.incrementSelectQueryCount(); - } catch (IgnoreMeException e) { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == TiDBOracleFactory.CERT)) { + // Disable strict Group By constraints for ROW oracle + globalState.executeStatement(new SQLQueryAdapter( + "SET @@sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';")); + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + TiDBErrors.addExpressionErrors(errors); + for (TiDBTable table : globalState.getSchema().getDatabaseTables()) { + if (!table.isView()) { + globalState.executeStatement(new SQLQueryAdapter("ANALYZE TABLE " + table.getName() + ";", errors)); + } } } - try { - if (globalState.getOptions().logEachSelect()) { - logger.getCurrentFileWriter().close(); - logger.currentFileWriter = null; + + // TiFlash replication settings + if (globalState.getDbmsSpecificOptions().tiflash) { + ExpectedErrors errors = new ExpectedErrors(); + TiDBErrors.addExpressionErrors(errors); + for (TiDBTable table : globalState.getSchema().getDatabaseTables()) { + if (!table.isView()) { + globalState.executeStatement( + new SQLQueryAdapter("ALTER TABLE " + table.getName() + " SET TIFLASH REPLICA 1;", errors)); + } + } + if (Randomly.getBoolean()) { + globalState.executeStatement(new SQLQueryAdapter("set @@tidb_enforce_mpp=1;")); } - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); } - } @Override - public Connection createDatabase(TiDBGlobalState globalState) throws SQLException { + public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLException { + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = TiDBOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = TiDBOptions.DEFAULT_PORT; + } + String databaseName = globalState.getDatabaseName(); - String url = "jdbc:mysql://127.0.0.1:4000/"; + String url = String.format("jdbc:mysql://%s:%d/", host, port); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - globalState.getState().statements.add(new QueryAdapter("USE test")); - globalState.getState().statements.add(new QueryAdapter("DROP DATABASE IF EXISTS " + databaseName)); + globalState.getState().logStatement("USE test"); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); String createDatabaseCommand = "CREATE DATABASE " + databaseName; - globalState.getState().statements.add(new QueryAdapter(createDatabaseCommand)); - globalState.getState().statements.add(new QueryAdapter("USE " + databaseName)); + globalState.getState().logStatement(createDatabaseCommand); + globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); + if (globalState.getDbmsSpecificOptions().nonPreparePlanCache) { + s.execute("set global tidb_enable_non_prepared_plan_cache=ON;"); + } } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); } con.close(); - con = DriverManager.getConnection("jdbc:mysql://127.0.0.1:4000/" + databaseName, - globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); - return con; + con = DriverManager.getConnection(url + databaseName, globalState.getOptions().getUserName(), + globalState.getOptions().getPassword()); + return new SQLConnection(con); } @Override @@ -219,4 +210,54 @@ public String getDBMSName() { return "tidb"; } + @Override + public String getQueryPlan(String selectStr, TiDBGlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN FORMAT=brief " + selectStr); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String targetQueryPlan = rs.getString(1).replace("├─", "").replace("└─", "").replace("│", "").trim() + + ";"; // Unify format + queryPlan += targetQueryPlan; + } + } + } catch (Throwable e) { + e.printStackTrace(); + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, TiDBGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + + @Override + public boolean addRowsToAllTables(TiDBGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (TiDBTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = TiDBInsertGenerator.getQuery(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index c502cc479..4ce7306e5 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -1,6 +1,5 @@ package sqlancer.tidb; -import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -8,14 +7,16 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.schema.AbstractSchema; -import sqlancer.schema.AbstractTable; -import sqlancer.schema.AbstractTableColumn; -import sqlancer.schema.AbstractTables; -import sqlancer.schema.TableIndex; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; -public class TiDBSchema extends AbstractSchema { +public class TiDBSchema extends AbstractSchema { public enum TiDBDataType { @@ -55,6 +56,23 @@ public boolean isNumeric() { throw new AssertionError(this); } } + + public boolean canHaveDefault() { + switch (this) { + case INT: + case DECIMAL: + case FLOATING: + case BOOL: + case CHAR: + return true; + case NUMERIC: + case TEXT: + case BLOB: + return false; + default: + throw new AssertionError(this); + } + } } public static class TiDBCompositeDataType { @@ -142,13 +160,17 @@ public static class TiDBColumn extends AbstractTableColumn { @@ -175,12 +201,21 @@ public TiDBTables getRandomTableNonEmptyTables() { return new TiDBTables(Randomly.nonEmptySubset(getDatabaseTables())); } + public int getIndexCount() { + int count = 0; + for (TiDBTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + private static TiDBCompositeDataType getColumnType(String typeString) { String trimmedStringType = typeString.replace(" zerofill", "").replace(" unsigned", ""); if (trimmedStringType.contains("decimal")) { return new TiDBCompositeDataType(TiDBDataType.DECIMAL); } - if (trimmedStringType.startsWith("var_string") || trimmedStringType.contains("binary")) { + if (trimmedStringType.startsWith("var_string") || trimmedStringType.contains("binary") + || trimmedStringType.startsWith("varchar")) { return new TiDBCompositeDataType(TiDBDataType.TEXT); } if (trimmedStringType.startsWith("char")) { @@ -194,35 +229,62 @@ private static TiDBCompositeDataType getColumnType(String typeString) { } else { switch (trimmedStringType) { case "text": + case "mediumtext": case "longtext": + case "tinytext": primitiveType = TiDBDataType.TEXT; break; case "float": + size = 4; + primitiveType = TiDBDataType.FLOATING; + break; case "double": + case "double(8,6)": // workaround to address https://github.com/sqlancer/sqlancer/issues/669 + case "double(23,16)": + size = 8; primitiveType = TiDBDataType.FLOATING; break; case "tinyint(1)": primitiveType = TiDBDataType.BOOL; + size = 1; break; case "null": primitiveType = TiDBDataType.INT; + size = 1; break; + case "tinyint": + case "tinyint(2)": + case "tinyint(3)": case "tinyint(4)": primitiveType = TiDBDataType.INT; size = 1; break; + case "smallint": + case "smallint(5)": case "smallint(6)": primitiveType = TiDBDataType.INT; size = 2; break; + case "int": + case "int(10)": case "int(11)": primitiveType = TiDBDataType.INT; size = 4; break; case "blob": + case "mediumblob": case "longblob": + case "tinyblob": primitiveType = TiDBDataType.BLOB; break; + case "date": + case "datetime": + case "datetime(6)": // workaround to address https://github.com/sqlancer/sqlancer/issues/669 + case "timestamp": + case "time": + case "year": + primitiveType = TiDBDataType.NUMERIC; + break; default: throw new AssertionError(trimmedStringType); } @@ -230,25 +292,25 @@ private static TiDBCompositeDataType getColumnType(String typeString) { return new TiDBCompositeDataType(primitiveType, size); } - public static class TiDBTable extends AbstractTable { + public static class TiDBTable extends AbstractRelationalTable { public TiDBTable(String tableName, List columns, List indexes, boolean isView) { super(tableName, columns, indexes, isView); } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } - public static TiDBSchema fromConnection(Connection con, String databaseName) throws SQLException { + public static TiDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); + // Ignore invalid views + if (databaseColumns.isEmpty()) { + continue; + } List indexes = getIndexes(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); TiDBTable t = new TiDBTable(tableName, databaseColumns, indexes, isView); for (TiDBColumn c : databaseColumns) { c.setTable(t); @@ -259,7 +321,7 @@ public static TiDBSchema fromConnection(Connection con, String databaseName) thr return new TiDBSchema(databaseTables); } - private static List getTableNames(Connection con) throws SQLException { + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { ResultSet tableRs = s.executeQuery("SHOW TABLES"); @@ -271,7 +333,7 @@ private static List getTableNames(Connection con) throws SQLException { return tableNames; } - private static List getIndexes(Connection con, String tableName) throws SQLException { + private static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format("SHOW INDEX FROM %s", tableName))) { @@ -284,7 +346,7 @@ private static List getIndexes(Connection con, String tableName) thr return indexes; } - private static List getTableColumns(Connection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SHOW COLUMNS FROM " + tableName)) { @@ -293,10 +355,13 @@ private static List getTableColumns(Connection con, String tableName String dataType = rs.getString("Type"); boolean isNullable = rs.getString("Null").contentEquals("YES"); boolean isPrimaryKey = rs.getString("Key").contains("PRI"); - TiDBColumn c = new TiDBColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + boolean hasDefault = rs.getString("Default") != null; + TiDBColumn c = new TiDBColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable, + hasDefault); columns.add(c); } } + } catch (SQLException e) { // Happens when } return columns; } diff --git a/src/sqlancer/tidb/ast/TiDBAggregate.java b/src/sqlancer/tidb/ast/TiDBAggregate.java index 5f5190f11..6650dc395 100644 --- a/src/sqlancer/tidb/ast/TiDBAggregate.java +++ b/src/sqlancer/tidb/ast/TiDBAggregate.java @@ -3,17 +3,13 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.ast.FunctionNode; +import sqlancer.common.ast.FunctionNode; import sqlancer.tidb.ast.TiDBAggregate.TiDBAggregateFunction; public class TiDBAggregate extends FunctionNode implements TiDBExpression { public enum TiDBAggregateFunction { - COUNT(1), // - SUM(1), // - AVG(1), // - MIN(1), // - MAX(1); + AVG(1), BIT_AND(1), BIT_OR(1), COUNT(1), SUM(1), MIN(1), MAX(1); private int nrArgs; diff --git a/src/sqlancer/tidb/ast/TiDBBinaryArithmeticOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryArithmeticOperation.java index 1669bac6c..e26284f6e 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryArithmeticOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryArithmeticOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBBinaryArithmeticOperation.TiDBBinaryArithmeticOperator; public class TiDBBinaryArithmeticOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBBinaryBitOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryBitOperation.java index a5127c99b..c97360dbd 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryBitOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryBitOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBBinaryBitOperation.TiDBBinaryBitOperator; public class TiDBBinaryBitOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java index 4d3e13ea3..c7240e4b7 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBBinaryComparisonOperation.TiDBComparisonOperator; public class TiDBBinaryComparisonOperation extends BinaryOperatorNode @@ -14,8 +14,8 @@ public enum TiDBComparisonOperator implements Operator { GREATER_EQUALS(">="), // SMALLER("<"), // SMALLER_EQUALS("<="), // - NOT_EQUALS("!="); // - // NULL_SAFE_EQUALS("<=>"); https://github.com/tidb-challenge-program/bug-hunting-issue/issues/5 + NOT_EQUALS("!="), // + NULL_SAFE_EQUALS("<=>"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/5 private String textRepr; diff --git a/src/sqlancer/tidb/ast/TiDBBinaryLogicalOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryLogicalOperation.java index 732d4b3ee..51553a373 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryLogicalOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryLogicalOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; public class TiDBBinaryLogicalOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBCollate.java b/src/sqlancer/tidb/ast/TiDBCollate.java index 1b74faa6d..08ae71bac 100644 --- a/src/sqlancer/tidb/ast/TiDBCollate.java +++ b/src/sqlancer/tidb/ast/TiDBCollate.java @@ -1,6 +1,6 @@ package sqlancer.tidb.ast; -import sqlancer.ast.UnaryNode; +import sqlancer.common.ast.UnaryNode; public class TiDBCollate extends UnaryNode implements TiDBExpression { diff --git a/src/sqlancer/tidb/ast/TiDBExpression.java b/src/sqlancer/tidb/ast/TiDBExpression.java index 49ac9fb70..1f4921836 100644 --- a/src/sqlancer/tidb/ast/TiDBExpression.java +++ b/src/sqlancer/tidb/ast/TiDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.tidb.ast; -public interface TiDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.tidb.TiDBSchema.TiDBColumn; + +public interface TiDBExpression extends Expression { } diff --git a/src/sqlancer/tidb/ast/TiDBFunctionCall.java b/src/sqlancer/tidb/ast/TiDBFunctionCall.java index 4aded9ad2..c7bdb2d99 100644 --- a/src/sqlancer/tidb/ast/TiDBFunctionCall.java +++ b/src/sqlancer/tidb/ast/TiDBFunctionCall.java @@ -95,8 +95,7 @@ public int getNrArgs() { RIGHT(2), // // RPAD TODO RTRIM(1), // - // SPACE(1), - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/6 + SPACE(1), // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/6 STRCMP(2), // SUBSTRING(2), // TODO: support other versions SUBSTRING_INDEX(3), // diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index e22b34b6a..1e9a30b63 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -1,27 +1,36 @@ package sqlancer.tidb.ast; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBTable; -public class TiDBJoin implements TiDBExpression { +public class TiDBJoin implements TiDBExpression, Join { private final TiDBExpression leftTable; private final TiDBExpression rightTable; - private final JoinType joinType; - private final TiDBExpression onCondition; + private JoinType joinType; + private TiDBExpression onCondition; private NaturalJoinType outerType; public enum JoinType { - INNER, NATURAL, STRAIGHT, LEFT, RIGHT; + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; public static JoinType getRandom() { return Randomly.fromOptions(values()); } + + public static JoinType getRandomExcept(JoinType... exclude) { + JoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(JoinType[]::new); + return Randomly.fromOptions(values); + } } public enum NaturalJoinType { @@ -52,10 +61,18 @@ public JoinType getJoinType() { return joinType; } + public void setJoinType(JoinType joinType) { + this.joinType = joinType; + } + public TiDBExpression getOnCondition() { return onCondition; } + public static TiDBJoin createCrossJoin(TiDBExpression left, TiDBExpression right, TiDBExpression onClause) { + return new TiDBJoin(left, right, JoinType.CROSS, onClause); + } + public static TiDBJoin createNaturalJoin(TiDBExpression left, TiDBExpression right, NaturalJoinType type) { TiDBJoin tiDBJoin = new TiDBJoin(left, right, JoinType.NATURAL, null); tiDBJoin.setNaturalJoinType(type); @@ -86,8 +103,8 @@ public NaturalJoinType getNaturalJoinType() { return outerType; } - public static List getJoins(List tableList, TiDBGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, TiDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBoolean()) { TiDBTableReference leftTable = (TiDBTableReference) tableList.remove(0); TiDBTableReference rightTable = (TiDBTableReference) tableList.remove(0); @@ -110,6 +127,9 @@ public static List getJoins(List tableList, TiDB case RIGHT: joinExpressions.add(TiDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); break; + case CROSS: + joinExpressions.add(TiDBJoin.createCrossJoin(leftTable, rightTable, null)); + break; default: throw new AssertionError(); } @@ -117,4 +137,45 @@ public static List getJoins(List tableList, TiDB return joinExpressions; } + public static List getJoinsWithoutNature(List tableList, + TiDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + TiDBTableReference leftTable = (TiDBTableReference) tableList.remove(0); + TiDBTableReference rightTable = (TiDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + TiDBExpressionGenerator joinGen = new TiDBExpressionGenerator(globalState).setColumns(columns); + switch (TiDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(TiDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case STRAIGHT: + joinExpressions.add(TiDBJoin.createStraightJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case LEFT: + joinExpressions.add(TiDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions.add(TiDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + case CROSS: + joinExpressions.add(TiDBJoin.createCrossJoin(leftTable, rightTable, null)); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public void setOnCondition(TiDBExpression generateExpression) { + this.onCondition = generateExpression; + } + + @Override + public void setOnClause(TiDBExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/tidb/ast/TiDBOrderingTerm.java b/src/sqlancer/tidb/ast/TiDBOrderingTerm.java index d11da943a..d735e142b 100644 --- a/src/sqlancer/tidb/ast/TiDBOrderingTerm.java +++ b/src/sqlancer/tidb/ast/TiDBOrderingTerm.java @@ -1,6 +1,6 @@ package sqlancer.tidb.ast; -import sqlancer.visitor.UnaryOperation; +import sqlancer.common.visitor.UnaryOperation; public class TiDBOrderingTerm implements UnaryOperation, TiDBExpression { diff --git a/src/sqlancer/tidb/ast/TiDBRegexOperation.java b/src/sqlancer/tidb/ast/TiDBRegexOperation.java index 07ce00eb0..f5f011dd6 100644 --- a/src/sqlancer/tidb/ast/TiDBRegexOperation.java +++ b/src/sqlancer/tidb/ast/TiDBRegexOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode; -import sqlancer.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.tidb.ast.TiDBRegexOperation.TiDBRegexOperator; public class TiDBRegexOperation extends BinaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBSelect.java b/src/sqlancer/tidb/ast/TiDBSelect.java index 54ccf9aa5..7da05da4c 100644 --- a/src/sqlancer/tidb/ast/TiDBSelect.java +++ b/src/sqlancer/tidb/ast/TiDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.tidb.ast; -import sqlancer.ast.SelectBase; +import java.util.List; +import java.util.stream.Collectors; -public class TiDBSelect extends SelectBase implements TiDBExpression { +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBTable; +import sqlancer.tidb.visitor.TiDBVisitor; + +public class TiDBSelect extends SelectBase + implements TiDBExpression, Select { private TiDBExpression hint; @@ -14,4 +22,20 @@ public TiDBExpression getHint() { return hint; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (TiDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (TiDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return TiDBVisitor.asString(this); + } } diff --git a/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java b/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java index 9b22e579b..3d7b1221d 100644 --- a/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java +++ b/src/sqlancer/tidb/ast/TiDBUnaryPostfixOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation.TiDBUnaryPostfixOperator; public class TiDBUnaryPostfixOperation extends UnaryOperatorNode diff --git a/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java b/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java index 52a81eb0a..c32c076ba 100644 --- a/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java +++ b/src/sqlancer/tidb/ast/TiDBUnaryPrefixOperation.java @@ -1,8 +1,8 @@ package sqlancer.tidb.ast; import sqlancer.Randomly; -import sqlancer.ast.BinaryOperatorNode.Operator; -import sqlancer.ast.UnaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; public class TiDBUnaryPrefixOperation extends UnaryOperatorNode diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 1878fa6d6..4d0807d4b 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -1,16 +1,14 @@ package sqlancer.tidb.gen; -import java.util.HashSet; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.tidb.TiDBBugs; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; import sqlancer.tidb.TiDBSchema.TiDBDataType; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -20,11 +18,22 @@ private TiDBAlterTableGenerator() { } private enum Action { - MODIFY_COLUMN, ENABLE_DISABLE_KEYS, FORCE, DROP_PRIMARY_KEY, ADD_PRIMARY_KEY, CHANGE, DROP_COLUMN, ORDER_BY + MODIFY_COLUMN, ENABLE_DISABLE_KEYS, DROP_PRIMARY_KEY, ADD_PRIMARY_KEY, CHANGE, DROP_COLUMN, ORDER_BY } - public static Query getQuery(TiDBGlobalState globalState) { - Set errors = new HashSet<>(); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add( + "Information schema is changed during the execution of the statement(for example, table definition may be updated by other DDL ran in parallel)"); + errors.add("Data truncat"); + errors.add("without a key length"); + errors.add("supported"); + errors.add("SQL syntax"); + errors.add("can't drop"); + errors.add("A PRIMARY must include all columns in the table's partitioning function"); + errors.add("key was too long"); + errors.add("Duplicate entry"); + errors.add("has a partitioning function dependency and cannot be dropped or renamed"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); @@ -33,14 +42,10 @@ public static Query getQuery(TiDBGlobalState globalState) { sb.append(" "); switch (a) { case MODIFY_COLUMN: - if (TiDBBugs.bug10) { - throw new IgnoreMeException(); - } sb.append("MODIFY "); sb.append(column.getName()); sb.append(" "); - sb.append(TiDBDataType.getRandom()); - errors.add("Unsupported modify column"); + sb.append(TiDBCompositeDataType.getRandom().toString()); break; case DROP_COLUMN: sb.append(" DROP "); @@ -48,7 +53,7 @@ public static Query getQuery(TiDBGlobalState globalState) { throw new IgnoreMeException(); } sb.append(column.getName()); - errors.add("with index covered now"); + errors.add("with composite index covered or Primary Key covered now"); errors.add("Unsupported drop integer primary key"); errors.add("has a generated column dependency"); errors.add( @@ -58,35 +63,41 @@ public static Query getQuery(TiDBGlobalState globalState) { sb.append(Randomly.fromOptions("ENABLE", "DISABLE")); sb.append(" KEYS"); break; - case FORCE: - sb.append("FORCE"); - break; case DROP_PRIMARY_KEY: if (!column.isPrimaryKey()) { throw new IgnoreMeException(); } errors.add("Unsupported drop integer primary key"); errors.add("Unsupported drop primary key when alter-primary-key is false"); + errors.add("Unsupported drop primary key when the table's pkIsHandle is true"); + errors.add("Incorrect table definition; there can be only one auto column and it must be defined as a key"); sb.append(" DROP PRIMARY KEY"); break; case ADD_PRIMARY_KEY: sb.append("ADD PRIMARY KEY("); - sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> c.getName()) - .collect(Collectors.joining(", "))); + sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> { + StringBuilder colName = new StringBuilder(c.getName()); + if (c.getType().getPrimitiveDataType() == TiDBDataType.TEXT + || c.getType().getPrimitiveDataType() == TiDBDataType.BLOB) { + TiDBTableGenerator.appendSpecifiers(colName, c.getType().getPrimitiveDataType()); + } + return colName; + }).collect(Collectors.joining(", "))); sb.append(")"); errors.add("Unsupported add primary key, alter-primary-key is false"); errors.add("Information schema is changed during the execution of the statement"); + errors.add("Multiple primary key defined"); + errors.add("Invalid use of NULL value"); + errors.add("Duplicate entry"); + errors.add("'Defining a virtual generated column as primary key' is not supported for generated columns"); break; case CHANGE: - if (TiDBBugs.bug10) { - throw new IgnoreMeException(); - } sb.append(" CHANGE "); sb.append(column.getName()); sb.append(" "); sb.append(column.getName()); sb.append(" "); - sb.append(column.getType().getPrimitiveDataType()); + sb.append(column.getType().toString()); sb.append(" NOT NULL "); errors.add("Invalid use of NULL value"); errors.add("Unsupported modify column:"); @@ -101,8 +112,7 @@ public static Query getQuery(TiDBGlobalState globalState) { default: throw new AssertionError(a); } - - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 6d3b87430..ede857dc5 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -1,11 +1,13 @@ package sqlancer.tidb.gen; import java.sql.SQLException; -import java.util.Arrays; +import java.util.List; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.TableIndex; +import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -14,30 +16,28 @@ public final class TiDBAnalyzeTableGenerator { private TiDBAnalyzeTableGenerator() { } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { + ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); - StringBuilder sb = new StringBuilder("ANALYZE "); - if (analyzeIndex && Randomly.getBoolean()) { - sb.append("INCREMENTAL "); - } - sb.append("TABLE "); + List indexes = table.getIndexes(); + indexes.removeIf(index -> index.getIndexName().contains("PRIMARY")); + boolean analyzeIndex = !indexes.isEmpty() && Randomly.getBoolean(); + StringBuilder sb = new StringBuilder("ANALYZE TABLE "); sb.append(table.getName()); if (analyzeIndex) { sb.append(" INDEX "); - sb.append(table.getRandomIndex().getIndexName()); + sb.append(Randomly.fromList(indexes).getIndexName()); + } + if (!analyzeIndex && Randomly.getBoolean()) { + sb.append(" ALL COLUMNS"); } if (Randomly.getBoolean()) { sb.append(" WITH "); sb.append(Randomly.getNotCachedInteger(1, 1024)); sb.append(" BUCKETS"); } - return new QueryAdapter(sb.toString(), Arrays.asList("https://github.com/pingcap/tidb/issues/15993", - /* https://github.com/pingcap/tidb/issues/15993 */ "doesn't have a default value" /* - * https://github. - * com/pingcap/tidb/ - * issues/15993 - */)); + errors.add("Fast analyze hasn't reached General Availability and only support analyze version 1 currently"); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index 5a41e74cf..c83d1a6eb 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -1,29 +1,34 @@ package sqlancer.tidb.gen; -import java.sql.SQLException; -import java.util.HashSet; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.visitor.TiDBVisitor; -public final class TiDBDeleteGenerator { +public final class TiDBDeleteGenerator extends AbstractDeleteGenerator { - private TiDBDeleteGenerator() { + private final TiDBGlobalState globalState; + + private TiDBDeleteGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + return new TiDBDeleteGenerator(globalState).getStatement(); } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { - Set errors = new HashSet<>(); + @Override + public void buildStatement() { + errors.addAll(TiDBErrors.getExpressionErrors()); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("DELETE "); + sb.append("DELETE "); if (Randomly.getBooleanWithSmallProbability()) { sb.append("LOW_PRIORITY "); } @@ -36,8 +41,7 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { sb.append("FROM "); sb.append(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(TiDBVisitor.asString(gen.generateExpression())); + appendWhereClause(TiDBVisitor.asString(gen.generateExpression())); errors.add("Truncated incorrect"); errors.add("Data truncation"); errors.add("Truncated incorrect FLOAT value"); @@ -49,14 +53,13 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { .collect(Collectors.joining(", "))); } if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); + appendLimitClause(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); } errors.add("Bad Number"); + errors.add("Truncated incorrect"); // https://github.com/pingcap/tidb/issues/24292 + errors.add("is not valid for CHARACTER SET"); errors.add("Division by 0"); errors.add("error parsing regexp"); - return new QueryAdapter(sb.toString(), errors); - } } diff --git a/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java b/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java new file mode 100644 index 000000000..727fffffc --- /dev/null +++ b/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.tidb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public final class TiDBDropTableGenerator { + + private TiDBDropTableGenerator() { + } + + public static SQLQueryAdapter dropTable(TiDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java new file mode 100644 index 000000000..486b5f873 --- /dev/null +++ b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.tidb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public final class TiDBDropViewGenerator { + + private TiDBDropViewGenerator() { + } + + public static SQLQueryAdapter dropView(TiDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> t.isView()).isEmpty()) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP VIEW "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index 7ee7505fa..f58695bb8 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -1,12 +1,13 @@ package sqlancer.tidb.gen; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.schema.TableIndex; -import sqlancer.tidb.TiDBBugs; +import sqlancer.common.schema.TableIndex; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.ast.TiDBSelect; import sqlancer.tidb.ast.TiDBText; @@ -23,15 +24,23 @@ enum IndexHint { INL_HASH_JOIN, // INL_MERGE_JOIN, // HASH_JOIN, // + READ_FROM_TIKV, // + READ_FROM_TIFLASH, // HASH_AGG, // STREAM_AGG, // USE_INDEX, // IGNORE_INDEX, // AGG_TO_COP, // - // READ_FROM_STORAGE USE_INDEX_MERGE, // NO_INDEX_MERGE, // - USE_TOJA; + USE_TOJA, // + HASH_JOIN_BUILD, // + HASH_JOIN_PROBE, // + MPP_1PHASE_AGG, // + MPP_2PHASE_AGG, // + LIMIT_TO_COP, // + SHUFFLE_JOIN, // + BROADCAST_JOIN } public TiDBHintGenerator(TiDBSelect select, List tables) { @@ -40,13 +49,40 @@ public TiDBHintGenerator(TiDBSelect select, List tables) { } public static void generateHints(TiDBSelect select, List tables) { - new TiDBHintGenerator(select, tables).generate(); + new TiDBHintGenerator(select, tables).randomHint(); + } + public static List generateAllHints(TiDBSelect select, List tables) { + TiDBHintGenerator generator = new TiDBHintGenerator(select, tables); + return generator.allHints(); } - private void generate() { + private void randomHint() { TiDBTable table = Randomly.fromList(tables); - switch (Randomly.fromOptions(IndexHint.values())) { + IndexHint chosenhint = Randomly.fromOptions(IndexHint.values()); + generate(table, chosenhint); + } + + private List allHints() { + List results = new ArrayList<>(); + IndexHint[] values = IndexHint.values(); + List availableHints = new ArrayList<>(Arrays.asList(values)); + + for (IndexHint hint : availableHints) { + try { + TiDBText generatedHint = generate(Randomly.fromList(tables), hint); + results.add(generatedHint); + } catch (IgnoreMeException e) { + continue; + } + } + return results; + } + + private TiDBText generate(TiDBTable table, IndexHint chosenhint) { + sb.setLength(0); + + switch (chosenhint) { case MERGE_JOIN: tablesHint("MERGE_JOIN"); break; @@ -54,9 +90,6 @@ private void generate() { tablesHint("INL_JOIN"); break; case INL_HASH_JOIN: - if (TiDBBugs.bug50) { - throw new IgnoreMeException(); - } tablesHint("INL_HASH_JOIN"); break; case INL_MERGE_JOIN: @@ -65,12 +98,27 @@ private void generate() { case HASH_JOIN: tablesHint("HASH_JOIN"); break; + case READ_FROM_TIKV: + storageHint("READ_FROM_STORAGE(TIKV"); + break; + case READ_FROM_TIFLASH: + storageHint("READ_FROM_STORAGE(TIFLASH"); + break; case HASH_AGG: sb.append("HASH_AGG()"); break; case STREAM_AGG: sb.append("STREAM_AGG()"); break; + case MPP_1PHASE_AGG: + sb.append("MPP_1PHASE_AGG()"); + break; + case MPP_2PHASE_AGG: + sb.append("MPP_2PHASE_AGG()"); + break; + case LIMIT_TO_COP: + sb.append("LIMIT_TO_COP()"); + break; case USE_INDEX: indexesHint("USE_INDEX"); break; @@ -80,16 +128,18 @@ private void generate() { case AGG_TO_COP: sb.append("AGG_TO_COP()"); break; + case SHUFFLE_JOIN: + twoTablesHint("SHUFFLE_JOIN", table); + break; case USE_INDEX_MERGE: - if (table.hasIndexes()) { - sb.append("USE_INDEX_MERGE("); - sb.append(table.getName()); - sb.append(", "); - List indexes = Randomly.nonEmptySubset(table.getIndexes()); - sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining(", "))); - sb.append(")"); + if (Randomly.getBoolean()) { + if (table.hasIndexes()) { + tablesHint("USE_INDEX_MERGE"); + } else { + throw new IgnoreMeException(); + } } else { - throw new IgnoreMeException(); + twoTablesHint("USE_INDEX_MERGE", table); } break; case NO_INDEX_MERGE: @@ -100,10 +150,21 @@ private void generate() { sb.append(Randomly.getBoolean()); sb.append(")"); break; + case HASH_JOIN_BUILD: + tablesHint("HASH_JOIN_BUILD"); + break; + case HASH_JOIN_PROBE: + tablesHint("HASH_JOIN_PROBE"); + break; + case BROADCAST_JOIN: + twoTablesHint("BROADCAST_JOIN", table); + break; default: throw new AssertionError(); } - select.setHint(new TiDBText(sb.toString())); + TiDBText hint = new TiDBText(sb.toString()); + select.setHint(hint); + return hint; } private void indexesHint(String string) { @@ -129,6 +190,27 @@ private void tablesHint(String string) { sb.append(")"); } + private void storageHint(String string) { + sb.append(string); + sb.append("["); + appendTables(); + sb.append("])"); + } + + private void twoTablesHint(String string, TiDBTable table) { + if (table.hasIndexes()) { + sb.append(string); + sb.append("("); + sb.append(table.getName()); + sb.append(", "); + List indexes = Randomly.nonEmptySubset(table.getIndexes()); + sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining(", "))); + sb.append(")"); + } else { + throw new IgnoreMeException(); + } + } + private void appendTables() { List tableSubset = Randomly.nonEmptySubset(tables); sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index a8f063e2e..64b4c808d 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -1,36 +1,43 @@ package sqlancer.tidb.gen; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBTable; -public final class TiDBIndexGenerator { +public class TiDBIndexGenerator extends AbstractIndexGenerator { - private TiDBIndexGenerator() { + private final TiDBGlobalState globalState; + + public TiDBIndexGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { - Set errors = new HashSet<>(); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } + return new TiDBIndexGenerator(globalState).getStatement(); + } + @Override + public void buildStatement() { TiDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - String indexName = globalState.getSchema().getFreeIndexName(); - StringBuilder sb = new StringBuilder("CREATE "); - if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append("UNIQUE "); + boolean unique = Randomly.getBooleanWithRatherLowProbability(); + if (unique) { errors.add("Duplicate for key"); errors.add("Duplicate entry "); errors.add("A UNIQUE INDEX must include all columns in the table's partitioning function"); } - sb.append("INDEX "); - sb.append(indexName); + appendCreateIndex(unique); + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); sb.append(randomTable.getName()); sb.append("("); @@ -59,7 +66,8 @@ public static Query getQuery(TiDBGlobalState globalState) throws SQLException { } errors.add("Cannot decode index value, because"); // invalid value for generated column errors.add("index already exist"); - return new QueryAdapter(sb.toString(), errors, true); + errors.add("Data truncation"); + errors.add("key was too long"); } } diff --git a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java index cf6bdbc12..363fa438b 100644 --- a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java @@ -1,14 +1,12 @@ package sqlancer.tidb.gen; import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -19,7 +17,7 @@ public class TiDBInsertGenerator { private final TiDBGlobalState globalState; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); private TiDBExpressionGenerator gen; public TiDBInsertGenerator(TiDBGlobalState globalState) { @@ -27,12 +25,16 @@ public TiDBInsertGenerator(TiDBGlobalState globalState) { TiDBErrors.addInsertErrors(errors); } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { - return new TiDBInsertGenerator(globalState).get(); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { + TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + return new TiDBInsertGenerator(globalState).get(table); } - private Query get() { - TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState, TiDBTable table) { + return new TiDBInsertGenerator(globalState).get(table); + } + + private SQLQueryAdapter get(TiDBTable table) { gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); StringBuilder sb = new StringBuilder(); boolean isInsert = Randomly.getBoolean(); @@ -68,7 +70,7 @@ private Query get() { sb.append(TiDBVisitor.asString(gen.generateExpression())); } errors.add("Illegal mix of collations"); - return new QueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), errors); } private void insertColumns(StringBuilder sb, List columns) { @@ -77,18 +79,14 @@ private void insertColumns(StringBuilder sb, List columns) { sb.append(", "); } sb.append("("); - for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { - if (nrColumn != 0) { + int i = 0; + for (TiDBColumn c : columns) { + if (i++ != 0) { sb.append(", "); } - insertValue(sb); + sb.append(TiDBVisitor.asString(gen.generateConstant(c.getType().getPrimitiveDataType()))); } sb.append(")"); } } - - private void insertValue(StringBuilder sb) { - sb.append(gen.generateConstant()); // TODO: try to insert valid data - } - } diff --git a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java index a2252c998..1b1599ac2 100644 --- a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java +++ b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java @@ -4,9 +4,8 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTables; @@ -20,9 +19,9 @@ public final class TiDBRandomQuerySynthesizer { private TiDBRandomQuerySynthesizer() { } - public static Query generate(TiDBGlobalState globalState, int nrColumns) { + public static SQLQueryAdapter generate(TiDBGlobalState globalState, int nrColumns) { TiDBSelect select = generateSelect(globalState, nrColumns); - return new QueryAdapter(TiDBVisitor.asString(select)); + return new SQLQueryAdapter(TiDBVisitor.asString(select)); } public static TiDBSelect generateSelect(TiDBGlobalState globalState, int nrColumns) { @@ -42,7 +41,7 @@ public static TiDBSelect generateSelect(TiDBGlobalState globalState, int nrColum select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 2cc2953e7..c997f6717 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -3,9 +3,8 @@ import java.sql.SQLException; import java.util.function.Function; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; public final class TiDBSetGenerator { @@ -44,11 +43,12 @@ private enum Action { TIDB_ENABLE_WINDOW_FUNCTION("tidb_enable_window_function", (r) -> Randomly.fromOptions(0, 1)), - TIDB_ENABLE_FAST_ANALYZE("tidb_enable_fast_analyze", (r) -> Randomly.fromOptions(0, 1)), // + // TIDB_ENABLE_FAST_ANALYZE("tidb_enable_fast_analyze", (r) -> Randomly.fromOptions(0, 1)), // + // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 + // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - // TODO: global - // TIDB_SCATTER_REGION("tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)); - TIDB_ENABLE_STMT_SUMMARY("tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("``", "`table`", "global")), + TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); private String name; @@ -61,14 +61,14 @@ private enum Action { } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { StringBuilder sb = new StringBuilder(); Action option = Randomly.fromOptions(Action.values()); sb.append("set @@"); sb.append(option.name); sb.append("="); sb.append(option.prod.apply(globalState.getRandomly())); - return new QueryAdapter(sb.toString()); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 5be1f4df1..451681409 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -2,16 +2,13 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; -import sqlancer.tidb.TiDBBugs; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; @@ -25,16 +22,24 @@ public class TiDBTableGenerator { private boolean allowPrimaryKey; private final List columns = new ArrayList<>(); private boolean primaryKeyAsTableConstraints; - private final Set errors = new HashSet<>(); + private final ExpectedErrors errors = new ExpectedErrors(); - public Query getQuery(TiDBGlobalState globalState) throws SQLException { + public static SQLQueryAdapter createRandomTableStatement(TiDBGlobalState globalState) throws SQLException { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return new TiDBTableGenerator().getQuery(globalState); + } + + public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { errors.add("Information schema is changed during the execution of the statement"); + errors.add("A CLUSTERED INDEX must include all columns in the table's partitioning function"); String tableName = globalState.getSchema().getFreeTableName(); int nrColumns = Randomly.smallNumber() + 1; allowPrimaryKey = Randomly.getBoolean(); primaryKeyAsTableConstraints = allowPrimaryKey && Randomly.getBoolean(); for (int i = 0; i < nrColumns; i++) { - TiDBColumn fakeColumn = new TiDBColumn("c" + i, null, false, false); + TiDBColumn fakeColumn = new TiDBColumn("c" + i, null, false, false, false); columns.add(fakeColumn); } TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(columns); @@ -42,14 +47,14 @@ public Query getQuery(TiDBGlobalState globalState) throws SQLException { StringBuilder sb = new StringBuilder("CREATE TABLE "); sb.append(tableName); - if (Randomly.getBoolean() && globalState.getSchema().getDatabaseTables().size() > 0) { + if (Randomly.getBoolean() && !globalState.getSchema().getDatabaseTables().isEmpty()) { sb.append(" LIKE "); TiDBTable otherTable = globalState.getSchema().getRandomTable(); sb.append(otherTable.getName()); } else { createNewTable(gen, sb); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { @@ -61,10 +66,7 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { sb.append(columns.get(i).getName()); sb.append(" "); TiDBCompositeDataType type; - do { - type = TiDBCompositeDataType.getRandom(); - } while (type.getPrimitiveDataType() == TiDBDataType.INT && type.getSize() < 4 - || type.getPrimitiveDataType() == TiDBDataType.BOOL); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/49 + type = TiDBCompositeDataType.getRandom(); appendType(sb, type); sb.append(" "); boolean isGeneratedColumn = Randomly.getBooleanWithRatherLowProbability(); @@ -74,7 +76,6 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { sb.append(") "); sb.append(Randomly.fromOptions("STORED", "VIRTUAL")); sb.append(" "); - errors.add("You have an error in your SQL syntax"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/53 errors.add("Generated column can refer only to generated columns defined prior to it"); errors.add( "'Defining a virtual generated column as primary key' is not supported for generated columns."); @@ -89,10 +90,9 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("NOT NULL "); } - if (Randomly.getBoolean() && type.getPrimitiveDataType() != TiDBDataType.TEXT - && type.getPrimitiveDataType() != TiDBDataType.BLOB && !isGeneratedColumn) { + if (Randomly.getBoolean() && type.getPrimitiveDataType().canHaveDefault() && !isGeneratedColumn) { sb.append("DEFAULT "); - sb.append(TiDBVisitor.asString(gen.generateConstant())); + sb.append(TiDBVisitor.asString(gen.generateConstant(type.getPrimitiveDataType()))); sb.append(" "); errors.add("Invalid default value"); errors.add( @@ -121,8 +121,7 @@ && canUseAsUnique(type) && !isGeneratedColumn) { errors.add(" used in key specification without a key length"); } sb.append(")"); - if (Randomly.getBooleanWithRatherLowProbability() - && !TiDBBugs.bug14 /* there are also a number of unresolved other partitioning bugs */) { + if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("PARTITION BY HASH("); sb.append(TiDBVisitor.asString(gen.generateExpression())); sb.append(") "); @@ -135,30 +134,6 @@ && canUseAsUnique(type) && !isGeneratedColumn) { errors.add("A UNIQUE INDEX must include all columns in the table's partitioning function"); errors.add("is of a not allowed type for this type of partitioning"); errors.add("The PARTITION function returns the wrong type"); - if (TiDBBugs.bug16) { - errors.add("UnknownType: *ast.WhenClause"); - } - } - List actions = Randomly.nonEmptySubset(Action.values()); - for (Action a : actions) { - sb.append(" "); - switch (a) { - case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - break; - case PRE_SPLIT_REGIONS: - sb.append("PRE_SPLIT_REGIONS="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - break; - case SHARD_ROW_ID_BITS: - sb.append("SHARD_ROW_ID_BITS="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - errors.add("Unsupported shard_row_id_bits for table with primary key as row id"); - break; - default: - throw new AssertionError(a); - } } } @@ -167,24 +142,16 @@ private boolean canUseAsUnique(TiDBCompositeDataType type) { } private void appendType(StringBuilder sb, TiDBCompositeDataType type) { - if (type.getPrimitiveDataType() == TiDBDataType.CHAR) { - throw new IgnoreMeException(); - } sb.append(type.toString()); appendSpecifiers(sb, type.getPrimitiveDataType()); appendSizeSpecifiers(sb, type.getPrimitiveDataType()); } - private enum Action { - AUTO_INCREMENT, PRE_SPLIT_REGIONS, SHARD_ROW_ID_BITS - } - private void appendSizeSpecifiers(StringBuilder sb, TiDBDataType type) { - if (type.isNumeric() && Randomly.getBoolean() && !TiDBBugs.bug16028) { + if (type.isNumeric() && Randomly.getBoolean()) { sb.append(" UNSIGNED"); } - if (type.isNumeric() && Randomly.getBoolean() - && !TiDBBugs.bug16028 /* seems to be the same bug as https://github.com/pingcap/tidb/issues/16028 */) { + if (type.isNumeric() && Randomly.getBoolean()) { sb.append(" ZEROFILL"); } } diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 7cb4c81ca..dd79670d9 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -1,13 +1,10 @@ package sqlancer.tidb.gen; -import java.sql.SQLException; -import java.util.HashSet; import java.util.List; -import java.util.Set; -import sqlancer.Query; -import sqlancer.QueryAdapter; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -15,41 +12,43 @@ import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.visitor.TiDBVisitor; -public final class TiDBUpdateGenerator { +public final class TiDBUpdateGenerator extends AbstractUpdateGenerator { - private TiDBUpdateGenerator() { + private final TiDBGlobalState globalState; + private TiDBExpressionGenerator gen; + + private TiDBUpdateGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + return new TiDBUpdateGenerator(globalState).getStatement(); } - public static Query getQuery(TiDBGlobalState globalState) throws SQLException { - Set errors = new HashSet<>(); + @Override + public void buildStatement() { TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("UPDATE "); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - if (Randomly.getBoolean()) { - sb.append(gen.generateConstant()); - } else { - sb.append(TiDBVisitor.asString(gen.generateExpression())); - TiDBErrors.addExpressionErrors(errors); - } - } + updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); - sb.append(TiDBVisitor.asString(gen.generateExpression())); - errors.add("Data Too Long"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/43 + appendWhereClause(TiDBVisitor.asString(gen.generateExpression())); } TiDBErrors.addInsertErrors(errors); + } - return new QueryAdapter(sb.toString(), errors); + @Override + protected void updateValue(TiDBColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant()); + } else { + sb.append(TiDBVisitor.asString(gen.generateExpression())); + TiDBErrors.addExpressionErrors(errors); + } } } diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index bbe336314..79c284ecd 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -1,20 +1,23 @@ package sqlancer.tidb.gen; -import java.util.HashSet; -import java.util.Set; - -import sqlancer.Query; -import sqlancer.QueryAdapter; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.ast.TiDBSelect; public final class TiDBViewGenerator { private TiDBViewGenerator() { } - public static Query getQuery(TiDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); if (Randomly.getBoolean()) { @@ -32,22 +35,25 @@ public static Query getQuery(TiDBGlobalState globalState) { if (i != 0) { sb.append(", "); } - sb.append("c" + i); + sb.append("c"); + sb.append(i); } sb.append(") AS "); - sb.append(TiDBRandomQuerySynthesizer.generate(globalState, nrColumns).getQueryString()); - Set errors = new HashSet<>(); + TiDBSelect select = TiDBRandomQuerySynthesizer.generateSelect(globalState, nrColumns); + if (TiDBBugs.bug38319 && !select.getGroupByExpressions().isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(select.asString()); + ExpectedErrors errors = new ExpectedErrors(); TiDBErrors.addExpressionErrors(errors); errors.add( "references invalid table(s) or column(s) or function(s) or definer/invoker of view lack rights to use them"); errors.add("Unknown column "); - if (Randomly.getBoolean()) { - sb.append(" WITH "); - sb.append(Randomly.fromOptions("CASCADED", "LOCAL")); - sb.append(" "); - sb.append(" CHECK OPTION"); + if (sb.toString().contains("\\\\")) { + // TODO: CREATE VIEW v0(c0) AS SELECT '\\' FROM t0; causes an unexpected failure + throw new IgnoreMeException(); } - return new QueryAdapter(sb.toString(), errors, true); + return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java new file mode 100644 index 000000000..57ee4409f --- /dev/null +++ b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java @@ -0,0 +1,80 @@ +package sqlancer.tidb.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.tidb.TiDBErrors; +import sqlancer.tidb.TiDBExpressionGenerator; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBTables; +import sqlancer.tidb.ast.TiDBColumnReference; +import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableReference; +import sqlancer.tidb.ast.TiDBText; +import sqlancer.tidb.gen.TiDBHintGenerator; +import sqlancer.tidb.visitor.TiDBVisitor; + +public class TiDBDQPOracle implements TestOracle { + private TiDBExpressionGenerator gen; + private final TiDBGlobalState state; + private TiDBSelect select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public TiDBDQPOracle(TiDBGlobalState globalState) { + state = globalState; + TiDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + // Randomly generate a query + TiDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new TiDBExpressionGenerator(state).setColumns(tables.getColumns()); + select = new TiDBSelect(); + + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new TiDBColumnReference(c)) + .collect(Collectors.toList())); + select.setFetchColumns(fetchColumns); + + List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + List joins = TiDBJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); + select.setJoinList(joins); + select.setFromList(tableList); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(gen.generateExpression()); + } + + String originalQueryString = TiDBVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + List hintList = TiDBHintGenerator.generateAllHints(select, tables.getTables()); + for (TiDBText hint : hintList) { + select.setHint(hint); + String queryString = TiDBVisitor.asString(select); + List result = ComparatorHelper.getResultSetFirstColumnAsString(queryString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, List.of(queryString), + state); + } + } + +} diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index 5ccc1d42d..d23e2c4c0 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -2,13 +2,13 @@ import java.sql.SQLException; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -20,27 +20,18 @@ import sqlancer.tidb.ast.TiDBJoin; import sqlancer.tidb.ast.TiDBSelect; import sqlancer.tidb.ast.TiDBTableReference; -import sqlancer.tidb.ast.TiDBUnaryPostfixOperation; -import sqlancer.tidb.ast.TiDBUnaryPostfixOperation.TiDBUnaryPostfixOperator; -import sqlancer.tidb.ast.TiDBUnaryPrefixOperation; -import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; import sqlancer.tidb.gen.TiDBHintGenerator; -public abstract class TiDBTLPBase implements TestOracle { - - final TiDBGlobalState state; - final Set errors = new HashSet<>(); +public abstract class TiDBTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { TiDBSchema s; TiDBTables targetTables; TiDBExpressionGenerator gen; TiDBSelect select; - TiDBExpression predicate; - TiDBExpression negatedPredicate; - TiDBExpression isNullPredicate; public TiDBTLPBase(TiDBGlobalState state) { - this.state = state; + super(state); TiDBErrors.addExpressionErrors(errors); } @@ -49,6 +40,7 @@ public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); gen = new TiDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); select = new TiDBSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); @@ -58,21 +50,19 @@ public void check() throws SQLException { List tableList = tables.stream().map(t -> new TiDBTableReference(t)) .collect(Collectors.toList()); - List joins = TiDBJoin.getJoins(tableList, state); + List joins = TiDBJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); select.setJoinList(joins); select.setFromList(tableList); select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new TiDBUnaryPrefixOperation(predicate, TiDBUnaryPrefixOperator.NOT); - isNullPredicate = new TiDBUnaryPostfixOperation(predicate, TiDBUnaryPostfixOperator.IS_NULL); } List generateFetchColumns() { return Arrays.asList(new TiDBColumnReference(targetTables.getColumns().get(0))); } - TiDBExpression generatePredicate() { - return gen.generateExpression(); + @Override + protected ExpressionGenerator getGen() { + return gen; } } diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index 74b92cd78..76e1a3c46 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -6,13 +6,15 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.TestOracle; +import sqlancer.common.oracle.TestOracle; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.ast.TiDBExpression; import sqlancer.tidb.visitor.TiDBVisitor; -public class TiDBTLPHavingOracle extends TiDBTLPBase implements TestOracle { +public class TiDBTLPHavingOracle extends TiDBTLPBase implements TestOracle { + + private String generatedQueryString; public TiDBTLPHavingOracle(TiDBGlobalState state) { super(state); @@ -27,11 +29,12 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); String originalQueryString = TiDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setHavingClause(predicate); @@ -48,7 +51,12 @@ public void check() throws SQLException { } @Override - TiDBExpression generatePredicate() { + protected TiDBExpression generatePredicate() { return gen.generateHavingClause(); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java deleted file mode 100644 index 4c8496e94..000000000 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.tidb.oracle; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.tidb.TiDBErrors; -import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.visitor.TiDBVisitor; - -public class TiDBTLPWhereOracle extends TiDBTLPBase { - - public TiDBTLPWhereOracle(TiDBGlobalState state) { - super(state); - TiDBErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = TiDBVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = TiDBVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = TiDBVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = TiDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - -} diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index ed396b591..67c7cda31 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -1,8 +1,7 @@ package sqlancer.tidb.visitor; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.tidb.TiDBBugs; +import sqlancer.common.visitor.ToStringVisitor; import sqlancer.tidb.ast.TiDBAggregate; import sqlancer.tidb.ast.TiDBCase; import sqlancer.tidb.ast.TiDBCastOperation; @@ -15,7 +14,6 @@ import sqlancer.tidb.ast.TiDBSelect; import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.ast.TiDBText; -import sqlancer.visitor.ToStringVisitor; public class TiDBToStringVisitor extends ToStringVisitor implements TiDBVisitor { @@ -76,9 +74,9 @@ public void visit(TiDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } } @@ -97,11 +95,7 @@ public void visit(TiDBJoin join) { sb.append(" "); switch (join.getJoinType()) { case INNER: - if (Randomly.getBoolean()) { - sb.append("INNER "); - } else { - sb.append("CROSS "); - } + sb.append("INNER "); sb.append("JOIN "); break; case LEFT: @@ -130,9 +124,6 @@ public void visit(TiDBJoin join) { sb.append("LEFT "); break; case RIGHT: - if (TiDBBugs.bug15844) { - throw new IgnoreMeException(); - } sb.append("RIGHT "); break; default: @@ -140,13 +131,15 @@ public void visit(TiDBJoin join) { } sb.append("JOIN "); break; + case CROSS: + sb.append("CROSS JOIN "); + break; default: throw new AssertionError(); } visit(join.getRightTable()); - sb.append(" "); - if (join.getJoinType() != JoinType.NATURAL) { - sb.append("ON "); + if (join.getOnCondition() != null && join.getJoinType() != JoinType.NATURAL) { + sb.append(" ON "); visit(join.getOnCondition()); } } diff --git a/src/sqlancer/transformations/JSQLParserBasedTransformation.java b/src/sqlancer/transformations/JSQLParserBasedTransformation.java new file mode 100644 index 000000000..e7b7b6145 --- /dev/null +++ b/src/sqlancer/transformations/JSQLParserBasedTransformation.java @@ -0,0 +1,36 @@ +package sqlancer.transformations; + +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; + +/** + * Transformations based on JSQLParser should be derived from this class. + */ + +public class JSQLParserBasedTransformation extends Transformation { + + protected Statement statement; + + public JSQLParserBasedTransformation(String desc) { + super(desc); + } + + @Override + protected void onStatementChanged() { + if (statementChangedHandler != null) { + statementChangedHandler.accept(this.statement.toString()); + } + } + + @Override + public boolean init(String sql) { + this.current = sql; + try { + statement = CCJSqlParserUtil.parse(current); + } catch (Exception e) { + return false; + } + return true; + } + +} diff --git a/src/sqlancer/transformations/RemoveClausesOfSelect.java b/src/sqlancer/transformations/RemoveClausesOfSelect.java new file mode 100644 index 000000000..832f82762 --- /dev/null +++ b/src/sqlancer/transformations/RemoveClausesOfSelect.java @@ -0,0 +1,102 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.statement.select.Distinct; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.Limit; +import net.sf.jsqlparser.statement.select.Offset; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SubSelect; +import net.sf.jsqlparser.statement.select.WithItem; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove clauses of a select, such as join, where, group by, distinct, offset, limit. + * + * e.g. select * from t where a = b offset 1 limit 1 -> select * from t; + */ + +public class RemoveClausesOfSelect extends JSQLParserBasedTransformation { + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + }; + + public RemoveClausesOfSelect() { + super("remove clauses of select"); + } + + @Override + public boolean init(String original) { + + boolean baseSuc = super.init(original); + if (!baseSuc) { + return false; + } + + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + + List withItemsList = select.getWithItemsList(); + if (withItemsList == null) { + return; + } + tryRemoveElms(select, withItemsList, Select::setWithItemsList); + + for (WithItem withItem : withItemsList) { + SubSelect subSelect = withItem.getSubSelect(); + if (subSelect == null) { + return; + } + + if (subSelect.getSelectBody() != null) { + subSelect.getSelectBody().accept(remover); + } + } + } + } + + private void handleSelect(PlainSelect plainSelect) { + + Expression where = plainSelect.getWhere(); + if (where != null) { + tryRemove(plainSelect, where, PlainSelect::setWhere); + } + + GroupByElement groupByElement = plainSelect.getGroupBy(); + if (groupByElement != null) { + tryRemove(plainSelect, groupByElement, PlainSelect::setGroupByElement); + } + + Distinct distinct = plainSelect.getDistinct(); + if (distinct != null) { + tryRemove(plainSelect, distinct, PlainSelect::setDistinct); + } + + Offset offset = plainSelect.getOffset(); + if (offset != null) { + tryRemove(plainSelect, offset, PlainSelect::setOffset); + } + + Limit limit = plainSelect.getLimit(); + if (offset != null) { + tryRemove(plainSelect, limit, PlainSelect::setLimit); + } + } + +} diff --git a/src/sqlancer/transformations/RemoveColumnsOfSelect.java b/src/sqlancer/transformations/RemoveColumnsOfSelect.java new file mode 100644 index 000000000..6cdd44d15 --- /dev/null +++ b/src/sqlancer/transformations/RemoveColumnsOfSelect.java @@ -0,0 +1,64 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SubSelect; +import net.sf.jsqlparser.statement.select.WithItem; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove columns of a select: e.g. select a, b, c from t -> select a from t. + */ +public class RemoveColumnsOfSelect extends JSQLParserBasedTransformation { + + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + tryRemoveElms(plainSelect, plainSelect.getSelectItems(), PlainSelect::setSelectItems); + super.visit(plainSelect); + } + }; + + public RemoveColumnsOfSelect() { + super("remove columns of a select"); + } + + @Override + public boolean init(String original) { + + boolean baseSucc = super.init(original); + if (!baseSucc) { + return false; + } + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + + List withItemsList = select.getWithItemsList(); + if (withItemsList == null) { + return; + } + for (WithItem withItem : withItemsList) { + SubSelect subSelect = withItem.getSubSelect(); + if (subSelect == null) { + return; + } + + if (subSelect.getSelectBody() != null) { + subSelect.getSelectBody().accept(remover); + } + } + + } + } +} diff --git a/src/sqlancer/transformations/RemoveElementsOfExpressionList.java b/src/sqlancer/transformations/RemoveElementsOfExpressionList.java new file mode 100644 index 000000000..ebeb0b0af --- /dev/null +++ b/src/sqlancer/transformations/RemoveElementsOfExpressionList.java @@ -0,0 +1,87 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.Join; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.InsertDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove elements of an expression list. + * + * NOTE: this only works for select statements and targets at ExpressionList type in JSQLParser, such as groupBy list + */ +public class RemoveElementsOfExpressionList extends JSQLParserBasedTransformation { + private final ExpressionDeParser expressionHandler = new ExpressionDeParser(); + private final SelectDeParser simplifier = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + + @Override + public void visit(ExpressionList expressionList) { + List expressions = expressionList.getExpressions(); + tryRemoveElms(expressionList, expressions, ExpressionList::setExpressions); + super.visit(expressionList); + } + }; + private final InsertDeParser insertDeParser = new InsertDeParser() { + @Override + public void visit(ExpressionList expressionList) { + List expressions = expressionList.getExpressions(); + tryRemoveElms(expressionList, expressions, ExpressionList::setExpressions); + super.visit(expressionList); + } + }; + + public RemoveElementsOfExpressionList() { + super("remove elements of expression lists"); + } + + @Override + public boolean init(String sql) { + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + this.simplifier.setExpressionVisitor(expressionHandler); + this.expressionHandler.setSelectVisitor(simplifier); + + this.insertDeParser.setExpressionVisitor(expressionHandler); + this.insertDeParser.setSelectVisitor(simplifier); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(simplifier); + } + } + + private void handleSelect(PlainSelect plainSelect) { + + GroupByElement groupByElement = plainSelect.getGroupBy(); + + if (groupByElement != null && groupByElement.getGroupByExpressionList() != null) { + ExpressionList expressionList = groupByElement.getGroupByExpressionList(); + List list = expressionList.getExpressions(); + tryRemoveElms(expressionList, list, ExpressionList::setExpressions); + } + + List expressionList = plainSelect.getJoins(); + if (expressionList != null) { + tryRemoveElms(plainSelect, expressionList, PlainSelect::setJoins); + } + } +} diff --git a/src/sqlancer/transformations/RemoveRowsOfInsert.java b/src/sqlancer/transformations/RemoveRowsOfInsert.java new file mode 100644 index 000000000..b49f80149 --- /dev/null +++ b/src/sqlancer/transformations/RemoveRowsOfInsert.java @@ -0,0 +1,45 @@ +package sqlancer.transformations; + +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.select.SelectBody; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.values.ValuesStatement; + +/** + * This Transformer remove rows of insert. Given a sql statement: + * + * INSERT INTO t1(c2, c0) VALUES (1508438260, 2929), (1508438260, TIMESTAMP '1969-12-26 01:57:21'), (0.5347171705591047, + * 398662142); -> INSERT INTO t1 (c2, c0) VALUES (0.5347171705591047, 398662142); + */ +public class RemoveRowsOfInsert extends JSQLParserBasedTransformation { + public RemoveRowsOfInsert() { + super("remove rows of an insert statement"); + } + + @Override + public void apply() { + super.apply(); + if (!(statement instanceof Insert)) { + return; + } + SelectBody selectBody = ((Insert) statement).getSelect().getSelectBody(); + if (!(selectBody instanceof SetOperationList)) { + return; + } + SetOperationList insertingList = (SetOperationList) selectBody; + for (SelectBody selBody : insertingList.getSelects()) { + if (!(selBody instanceof ValuesStatement)) { + continue; + } + ValuesStatement valuesStatement = (ValuesStatement) selBody; + ItemsList itemsList = valuesStatement.getExpressions(); + if (!(itemsList instanceof ExpressionList)) { + continue; + } + tryRemoveElms((ExpressionList) itemsList, ((ExpressionList) itemsList).getExpressions(), + ExpressionList::setExpressions); + } + } +} diff --git a/src/sqlancer/transformations/RemoveUnions.java b/src/sqlancer/transformations/RemoveUnions.java new file mode 100644 index 000000000..4f9593345 --- /dev/null +++ b/src/sqlancer/transformations/RemoveUnions.java @@ -0,0 +1,52 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectBody; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * try removing sub selects of a union statement. + * + * e.g. select 1 union select 2 -> select 1 + */ + +public class RemoveUnions extends JSQLParserBasedTransformation { + + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(SetOperationList list) { + List selectBodyList = list.getSelects(); + tryRemoveElms(list, selectBodyList, SetOperationList::setSelects); + super.visit(list); + } + }; + + public RemoveUnions() { + super("remove union selects"); + } + + @Override + public boolean init(String sql) { + + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + } + } +} diff --git a/src/sqlancer/transformations/RoundDoubleConstant.java b/src/sqlancer/transformations/RoundDoubleConstant.java new file mode 100644 index 000000000..d22496c4e --- /dev/null +++ b/src/sqlancer/transformations/RoundDoubleConstant.java @@ -0,0 +1,74 @@ +package sqlancer.transformations; + +import java.text.DecimalFormat; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Round double values which are longer than a certain length. e.g. 2.4782565267 -> 2.478. + * + * This transformation is not based on JSQLParser. + */ +public class RoundDoubleConstant extends Transformation { + + private Set doubleValueCollector; + + private String currentString; + + private static final int ROUND_LENGTH = 3; + private DecimalFormat decimalFormat; + + public RoundDoubleConstant() { + super("round double constant values"); + } + + @Override + public boolean init(String sql) { + super.init(sql); + decimalFormat = new DecimalFormat("#." + "#".repeat(ROUND_LENGTH)); + + currentString = sql; + doubleValueCollector = new HashSet<>(); + + String regex = "\\b-?\\d+\\.\\d+\\b"; + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(sql); + + while (matcher.find()) { + String matchedText = matcher.group(); + String decimalPart = matchedText.replaceAll("\\d+\\.", ""); + int decimalPlaces = decimalPart.length(); + if (decimalPlaces > ROUND_LENGTH) { + doubleValueCollector.add(matchedText); + } + } + return true; + } + + @Override + public void apply() { + for (String doubleValue : doubleValueCollector) { + + double targetNumber = Double.parseDouble(doubleValue); + String roundedNumberStr = decimalFormat.format(targetNumber); + + String replacement = currentString.replace(doubleValue, roundedNumberStr); + String original = currentString; + + tryReplace(null, original, replacement, (p, r) -> { + currentString = r; + }); + } + super.apply(); + } + + @Override + protected void onStatementChanged() { + if (statementChangedHandler != null) { + statementChangedHandler.accept(currentString); + } + } +} diff --git a/src/sqlancer/transformations/SimplifyConstant.java b/src/sqlancer/transformations/SimplifyConstant.java new file mode 100644 index 000000000..6638dca73 --- /dev/null +++ b/src/sqlancer/transformations/SimplifyConstant.java @@ -0,0 +1,103 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; + +import net.sf.jsqlparser.expression.DoubleValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.statement.StatementVisitorAdapter; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * Shorten the constant of a statement e.g. "a_very_long_str" -> "_", 12341234->1. + * + * Note: The API of JSQLParser may have some problems with double values: `setValue` can't change the literal value of a + * DoubleValue object. Therefore, double values are handled at RoundDoubleConstant class. + */ +public class SimplifyConstant extends JSQLParserBasedTransformation { + static class ConstantCollector extends ExpressionDeParser { + private final List candidates = new ArrayList<>(); + + @Override + public void visit(DoubleValue doubleValue) { + candidates.add(doubleValue); + super.visit(doubleValue); + } + + @Override + public void visit(LongValue longValue) { + candidates.add(longValue); + super.visit(longValue); + } + + @Override + public void visit(StringValue stringValue) { + candidates.add(stringValue); + super.visit(stringValue); + } + + public List getCandidates() { + return candidates; + } + } + + public SimplifyConstant() { + super("simplify constant expressions"); + } + + @Override + public void apply() { + super.apply(); + ConstantCollector collector = new ConstantCollector(); + StringBuilder buffer = new StringBuilder(); + SelectDeParser collectorDeParser = new SelectDeParser(collector, buffer); + collector.setSelectVisitor(collectorDeParser); + collector.setBuffer(buffer); + + List candidates = collector.getCandidates(); + + StatementVisitorAdapter statementVisitor = new StatementVisitorAdapter() { + @Override + public void visit(Insert insert) { + insert.getSelect().getSelectBody().accept(collectorDeParser); + super.visit(insert); + } + + @Override + public void visit(Select select) { + select.getSelectBody().accept(collectorDeParser); + super.visit(select); + } + }; + + statement.accept(statementVisitor); + + for (Expression e : candidates) { + if (e instanceof LongValue) { + simplify((LongValue) e); + } else if (e instanceof StringValue) { + simplify((StringValue) e); + } + } + } + + private void simplify(LongValue longValue) { + long variant = 0; + if (!longValue.getStringValue().equals(String.valueOf(variant))) { + tryReplace(longValue, longValue.getStringValue(), String.valueOf(variant), LongValue::setStringValue); + } + } + + private void simplify(StringValue stringValue) { + String variant = "_"; + if (!stringValue.getValue().equals(variant)) { + tryReplace(stringValue, stringValue.getValue(), variant, StringValue::setValue); + } + } + +} diff --git a/src/sqlancer/transformations/SimplifyExpressions.java b/src/sqlancer/transformations/SimplifyExpressions.java new file mode 100644 index 000000000..244629b4d --- /dev/null +++ b/src/sqlancer/transformations/SimplifyExpressions.java @@ -0,0 +1,98 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; + +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * This transformation simplifies complicated expressions e.g: a + (b + c) -> b. + */ + +public class SimplifyExpressions extends JSQLParserBasedTransformation { + public SimplifyExpressions() { + super("simplify expressions. e.g. a + b -> a"); + } + + @Override + public boolean init(String sql) { + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + this.simplifier.setExpressionVisitor(expressionHandler); + this.expressionHandler.setSelectVisitor(simplifier); + return true; + } + + private final ExpressionDeParser expressionHandler = new ExpressionDeParser() { + @Override + protected void visitBinaryExpression(BinaryExpression binaryExpression, String operator) { + + Expression lhs = binaryExpression.getLeftExpression(); + Expression rhs = binaryExpression.getRightExpression(); + + handleExpression(binaryExpression, lhs, BinaryExpression::setLeftExpression); + handleExpression(binaryExpression, rhs, BinaryExpression::setRightExpression); + + super.visitBinaryExpression(binaryExpression, operator); + } + + }; + private final SelectDeParser simplifier = new SelectDeParser() { + + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + }; + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(simplifier); + } + } + + private void handleSelect(PlainSelect plainSelect) { + Expression where = plainSelect.getWhere(); + if (where != null) { + handleExpression(plainSelect, where, PlainSelect::setWhere); + } + Expression having = plainSelect.getHaving(); + if (having != null) { + handleExpression(plainSelect, having, PlainSelect::setHaving); + } + } + + private List flattenExpression(Expression expression) { + if (expression instanceof BinaryExpression) { + BinaryExpression binaryExpression = (BinaryExpression) expression; + return List.of(binaryExpression.getLeftExpression(), binaryExpression.getRightExpression()); + } else if (expression instanceof Parenthesis) { + return List.of(((Parenthesis) expression).getExpression()); + } + return new ArrayList<>(); + } + + private

void handleExpression(P parent, Expression expr, BiConsumer setter) { + + List expressions = flattenExpression(expr); + for (Expression variant : expressions) { + boolean suc = tryReplace(parent, expr, variant, setter); + if (suc) { + break; + } + } + } +} diff --git a/src/sqlancer/transformations/Transformation.java b/src/sqlancer/transformations/Transformation.java new file mode 100644 index 000000000..affa4c281 --- /dev/null +++ b/src/sqlancer/transformations/Transformation.java @@ -0,0 +1,122 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * The base class of transformations. Defines APIs to remove, replace, remove elements of a list. + */ +public class Transformation { + + private static Supplier bugJudgement; + private static long reduceSteps; + + protected boolean isChanged; + protected String current; + protected String desc = ""; + + protected Consumer statementChangedHandler; + + public Transformation(String desc) { + this.desc = desc; + } + + @SuppressWarnings("unused") + protected Transformation() { + } + + public static void setBugJudgement(Supplier judgement) { + bugJudgement = judgement; + } + + @Override + public String toString() { + return desc; + } + + public boolean init(String sql) { + isChanged = false; + return true; + } + + public boolean tryRemove(P parent, T target, BiConsumer setter) { + setter.accept(parent, null); + onStatementChanged(); + if (!bugStillTriggers()) { + setter.accept(parent, target); + onStatementChanged(); + return false; + } + reduceSteps++; + isChanged = true; + return true; + } + + public boolean tryReplace(P parent, T original, T vari, BiConsumer setter) { + setter.accept(parent, vari); + onStatementChanged(); + if (!bugStillTriggers()) { + setter.accept(parent, original); + onStatementChanged(); + return false; + } + reduceSteps++; + isChanged = true; + return true; + } + + public void tryRemoveElms(P parent, List elms, // NOPMD + BiConsumer> setter) { + boolean observeChange; + do { + observeChange = false; + for (int i = elms.size() - 1; i >= 0; i--) { + List reducedElms = new ArrayList<>(elms); + reducedElms.subList(i, i + 1).clear(); + setter.accept(parent, reducedElms); + onStatementChanged(); + if (bugStillTriggers()) { + elms = reducedElms; + onStatementChanged(); + observeChange = true; + } + } + isChanged |= observeChange; + setter.accept(parent, elms); + reduceSteps++; + onStatementChanged(); + } while (observeChange); + + } + + public boolean bugStillTriggers() { + try { + return Transformation.bugJudgement.get(); + } catch (Exception ignored) { + } + return false; + } + + public void apply() { + isChanged = false; + } + + public boolean changed() { + return isChanged; + } + + public static long getReduceSteps() { + return reduceSteps; + } + + protected void onStatementChanged() { + } + + public void setStatementChangedCallBack(Consumer statementChangedHandler) { + this.statementChangedHandler = statementChangedHandler; + } + +} diff --git a/src/sqlancer/yugabyte/YugabyteBugs.java b/src/sqlancer/yugabyte/YugabyteBugs.java new file mode 100644 index 000000000..713ae6f88 --- /dev/null +++ b/src/sqlancer/yugabyte/YugabyteBugs.java @@ -0,0 +1,14 @@ +package sqlancer.yugabyte; + +public final class YugabyteBugs { + + // https://github.com/yugabyte/yugabyte-db/issues/11357 + public static boolean bug11357 = true; + + // https://github.com/yugabyte/yugabyte-db/issues/14330 + public static boolean bug14330 = true; + + private YugabyteBugs() { + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLErrors.java b/src/sqlancer/yugabyte/ycql/YCQLErrors.java new file mode 100644 index 000000000..010b2111a --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLErrors.java @@ -0,0 +1,32 @@ +package sqlancer.yugabyte.ycql; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class YCQLErrors { + + private YCQLErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Signature mismatch in call to builtin function"); + errors.add("Qualified name not allowed for column reference"); + errors.add("Datatype Mismatch"); + errors.add("Invalid Datatype"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Order by clause contains invalid expression"); + errors.add("Invalid Function Call"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java new file mode 100644 index 000000000..9d865aca2 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -0,0 +1,36 @@ +package sqlancer.yugabyte.ycql; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(separators = "=", commandDescription = "YCQL (default port: " + YCQLOptions.DEFAULT_PORT + + ", default host: " + YCQLOptions.DEFAULT_HOST) +public class YCQLOptions implements DBMSSpecificOptions { + + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 9042; + public static final String DEFAULT_DATACENTER = "datacenter1"; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--datacenter", description = "YCQL datacenter, can be found in system.local table", arity = 1) + public String datacenter = DEFAULT_DATACENTER; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(YCQLOracleFactory.FUZZER); + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java b/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java new file mode 100644 index 000000000..beaf43704 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java @@ -0,0 +1,18 @@ +package sqlancer.yugabyte.ycql; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ycql.test.YCQLFuzzer; + +public enum YCQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YCQLProvider.YCQLGlobalState globalState) + throws SQLException { + return new YCQLFuzzer(globalState); + } + + } +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java new file mode 100644 index 000000000..b82f5b0a2 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -0,0 +1,170 @@ +package sqlancer.yugabyte.ycql; + +import static sqlancer.yugabyte.ycql.YCQLSchema.getTableNames; +import static sqlancer.yugabyte.ysql.YSQLProvider.DDL_LOCK; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.gen.YCQLAlterTableGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLDeleteGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLIndexGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLInsertGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLRandomQuerySynthesizer; +import sqlancer.yugabyte.ycql.gen.YCQLTableGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLUpdateGenerator; + +@AutoService(DatabaseProvider.class) +public class YCQLProvider extends SQLProviderAdapter { + + public YCQLProvider() { + super(YCQLGlobalState.class, YCQLOptions.class); + } + + public enum Action implements AbstractAction { + + ALTER(YCQLAlterTableGenerator::getQuery), // + INSERT(YCQLInsertGenerator::getQuery), // + CREATE_INDEX(YCQLIndexGenerator::getQuery), // + DELETE(YCQLDeleteGenerator::generate), // + UPDATE(YCQLUpdateGenerator::getQuery), // + EXPLAIN((g) -> { + ExpectedErrors errors = new ExpectedErrors(); + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter( + "EXPLAIN " + YCQLToStringVisitor + .asString(YCQLRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + errors); + }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(YCQLGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(YCQLGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case ALTER: + return r.getInteger(0, 10); + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case CREATE_INDEX: + case UPDATE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + case EXPLAIN: + return r.getInteger(0, 2); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + default: + throw new AssertionError(a); + } + } + + public static class YCQLGlobalState extends SQLGlobalState { + + @Override + protected YCQLSchema readSchema() throws SQLException { + return YCQLSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(YCQLGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new YCQLTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + YCQLProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLException { + try { + Class.forName("com.ing.data.cassandra.jdbc.CassandraDriver"); + } catch (ClassNotFoundException e) { + throw new AssertionError(); + } + + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + + if (host == null) { + host = YCQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = YCQLOptions.DEFAULT_PORT; + } + + final String url = "jdbc:cassandra://%s:%s/%s?localdatacenter=%s"; + final Connection connection = DriverManager.getConnection( + String.format(url, host, port, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); + + synchronized (DDL_LOCK) { + try (Statement stmt = connection.createStatement()) { + try { + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); + } catch (Exception se) { + // try again + List tableNames = getTableNames( + new SQLConnection(DriverManager.getConnection(String.format(url, host, port, + globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), + globalState.getDatabaseName()); + for (String tableName : tableNames) { + stmt.execute("DROP TABLE " + globalState.getDatabaseName() + "." + tableName); + } + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); + } + + stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); + } + } + + return new SQLConnection(DriverManager.getConnection(String.format(url, host, port, + globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))); + } + + @Override + public String getDBMSName() { + return "ycql"; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java new file mode 100644 index 000000000..736247364 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -0,0 +1,263 @@ +package sqlancer.yugabyte.ycql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; + +public class YCQLSchema extends AbstractSchema { + + public enum YCQLDataType { + + INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP; + + public static YCQLDataType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public static class YCQLCompositeDataType { + + private final YCQLDataType dataType; + + private final int size; + + public YCQLCompositeDataType(YCQLDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public YCQLDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static YCQLCompositeDataType getRandom() { + YCQLDataType type = YCQLDataType.getRandom(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case BOOLEAN: + case VARCHAR: + case DATE: + case TIMESTAMP: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new YCQLCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return Randomly.fromOptions("BIGINT"); + case 4: + return Randomly.fromOptions("INTEGER", "INT"); + case 2: + return Randomly.fromOptions("SMALLINT"); + case 1: + return Randomly.fromOptions("TINYINT"); + default: + throw new AssertionError(size); + } + case VARCHAR: + return "VARCHAR"; + case FLOAT: + switch (size) { + case 8: + return Randomly.fromOptions("DOUBLE"); + case 4: + return Randomly.fromOptions("FLOAT"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN"); + case TIMESTAMP: + return Randomly.fromOptions("TIMESTAMP"); + case DATE: + return Randomly.fromOptions("DATE"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class YCQLColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public YCQLColumn(String name, YCQLCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + @Override + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class YCQLTables extends AbstractTables { + + public YCQLTables(List tables) { + super(tables); + } + + } + + public YCQLSchema(List databaseTables) { + super(databaseTables); + } + + public YCQLTables getRandomTableNonEmptyTables() { + return new YCQLTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + private static YCQLCompositeDataType getColumnType(String typeString) { + YCQLDataType primitiveType; + int size = -1; + switch (typeString.toUpperCase()) { + case "INT": + case "INTEGER": + primitiveType = YCQLDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = YCQLDataType.INT; + size = 2; + break; + case "BIGINT": + primitiveType = YCQLDataType.INT; + size = 8; + break; + case "TINYINT": + primitiveType = YCQLDataType.INT; + size = 1; + break; + case "VARCHAR": + case "TEXT": + primitiveType = YCQLDataType.VARCHAR; + break; + case "FLOAT": + primitiveType = YCQLDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + primitiveType = YCQLDataType.FLOAT; + size = 8; + break; + case "BOOLEAN": + primitiveType = YCQLDataType.BOOLEAN; + break; + case "DATE": + primitiveType = YCQLDataType.DATE; + break; + case "TIMESTAMP": + primitiveType = YCQLDataType.TIMESTAMP; + break; + default: + throw new AssertionError(); + } + return new YCQLCompositeDataType(primitiveType, size); + } + + public static class YCQLTable extends AbstractRelationalTable { + + public YCQLTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static YCQLSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con, databaseName); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; + } + List databaseColumns = getTableColumns(con, databaseName, tableName); + boolean isView = matchesViewName(tableName); + YCQLTable t = new YCQLTable(tableName, databaseColumns, isView); + for (YCQLColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new YCQLSchema(databaseTables); + } + + public static List getTableNames(SQLConnection con, String databaseName) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + String.format("select * from system_schema.tables where keyspace_name = '%s'", databaseName))) { + while (rs.next()) { + tableNames.add(rs.getString("table_name")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "select * from system_schema.columns where keyspace_name = '%s' and table_name = '%s'", + databaseName, tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("type"); + boolean isPrimaryKey = rs.getString("kind").contentEquals("partition_key"); + YCQLColumn c = new YCQLColumn(columnName, getColumnType(dataType), isPrimaryKey, true); + columns.add(c); + } + } + } + return columns; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java new file mode 100644 index 000000000..c9f357fe9 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java @@ -0,0 +1,63 @@ +package sqlancer.yugabyte.ycql; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLSelect; + +public class YCQLToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(YCQLExpression expr) { + if (expr instanceof YCQLConstant) { + visit((YCQLConstant) expr); + } else if (expr instanceof YCQLSelect) { + visit((YCQLSelect) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(YCQLConstant constant) { + sb.append(constant.toString()); + } + + private void visit(YCQLSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(YCQLExpression expr) { + YCQLToStringVisitor visitor = new YCQLToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java new file mode 100644 index 000000000..880007cd8 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java @@ -0,0 +1,9 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class YCQLBetweenOperation extends NewBetweenOperatorNode implements YCQLExpression { + public YCQLBetweenOperation(YCQLExpression left, YCQLExpression middle, YCQLExpression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java new file mode 100644 index 000000000..fac2af681 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class YCQLBinaryOperation extends NewBinaryOperatorNode implements YCQLExpression { + public YCQLBinaryOperation(YCQLExpression left, YCQLExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java new file mode 100644 index 000000000..9c81e70a5 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.yugabyte.ycql.YCQLSchema; + +public class YCQLColumnReference extends ColumnReferenceNode + implements YCQLExpression { + public YCQLColumnReference(YCQLSchema.YCQLColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java new file mode 100644 index 000000000..08f04bcfb --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java @@ -0,0 +1,171 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public class YCQLConstant implements YCQLExpression { + + private YCQLConstant() { + } + + public static class YCQLNullConstant extends YCQLConstant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class YCQLIntConstant extends YCQLConstant { + + private final long value; + + public YCQLIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class YCQLDoubleConstant extends YCQLConstant { + + private final double value; + + public YCQLDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static class YCQLTextConstant extends YCQLConstant { + + private final String value; + + public YCQLTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class YCQLDateConstant extends YCQLConstant { + + public String textRepr; + + public YCQLDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("'%s'", textRepr); + } + + } + + public static class YCQLTimestampConstant extends YCQLConstant { + + public String textRepr; + + public YCQLTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("'%s'", textRepr); + } + + } + + public static class YCQLBooleanConstant extends YCQLConstant { + + private final boolean value; + + public YCQLBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static YCQLExpression createStringConstant(String text) { + return new YCQLTextConstant(text); + } + + public static YCQLExpression createFloatConstant(double val) { + return new YCQLDoubleConstant(val); + } + + public static YCQLExpression createIntConstant(long val) { + return new YCQLIntConstant(val); + } + + public static YCQLExpression createNullConstant() { + return new YCQLNullConstant(); + } + + public static YCQLExpression createBooleanConstant(boolean val) { + return new YCQLBooleanConstant(val); + } + + public static YCQLExpression createDateConstant(long integer) { + return new YCQLDateConstant(integer); + } + + public static YCQLExpression createTimestampConstant(long integer) { + return new YCQLTimestampConstant(integer); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java b/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java new file mode 100644 index 000000000..4177344eb --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java @@ -0,0 +1,8 @@ +package sqlancer.yugabyte.ycql.ast; + +public interface YCQLExpression { + + default YCQLConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java new file mode 100644 index 000000000..483fab4c4 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java @@ -0,0 +1,11 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class YCQLFunction extends NewFunctionNode implements YCQLExpression { + public YCQLFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java new file mode 100644 index 000000000..49d43be11 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java @@ -0,0 +1,11 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class YCQLInOperation extends NewInOperatorNode implements YCQLExpression { + public YCQLInOperation(YCQLExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java new file mode 100644 index 000000000..5f647e721 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java @@ -0,0 +1,9 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class YCQLOrderingTerm extends NewOrderingTerm implements YCQLExpression { + public YCQLOrderingTerm(YCQLExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java new file mode 100644 index 000000000..25257057e --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java @@ -0,0 +1,17 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.SelectBase; + +public class YCQLSelect extends SelectBase implements YCQLExpression { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java new file mode 100644 index 000000000..70c01b820 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.yugabyte.ycql.YCQLSchema; + +public class YCQLTableReference extends TableReferenceNode + implements YCQLExpression { + public YCQLTableReference(YCQLSchema.YCQLTable table) { + super(table); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java new file mode 100644 index 000000000..b9165211a --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java @@ -0,0 +1,10 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class YCQLUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements YCQLExpression { + public YCQLUnaryPostfixOperation(YCQLExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java new file mode 100644 index 000000000..bd3b554ee --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java @@ -0,0 +1,10 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class YCQLUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements YCQLExpression { + public YCQLUnaryPrefixOperation(YCQLExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java new file mode 100644 index 000000000..d15d7fe1c --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java @@ -0,0 +1,48 @@ +package sqlancer.yugabyte.ycql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; + +public final class YCQLAlterTableGenerator { + + private YCQLAlterTableGenerator() { + } + + enum Action { + ADD_COLUMN, DROP_COLUMN + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(YCQLCompositeDataType.getRandom().toString()); + break; + case DROP_COLUMN: + sb.append("DROP "); + sb.append(table.getRandomColumn().getName()); + break; + default: + throw new AssertionError(action); + } + + errors.add("Alter key column. Can't alter key column"); + errors.add("cannot remove a key column"); + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java new file mode 100644 index 000000000..af99ae4a5 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -0,0 +1,34 @@ +package sqlancer.yugabyte.ycql.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; + +public final class YCQLDeleteGenerator extends AbstractDeleteGenerator { + + private final YCQLGlobalState globalState; + + private YCQLDeleteGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(YCQLGlobalState globalState) { + return new YCQLDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + appendDeleteFromTable(table.getName()); + if (Randomly.getBoolean()) { + appendWhereClause(YCQLToStringVisitor.asString( + new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); + } + YCQLErrors.addExpressionErrors(errors); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java new file mode 100644 index 000000000..0b0e4dee5 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -0,0 +1,300 @@ +package sqlancer.yugabyte.ycql.gen; + +import static sqlancer.yugabyte.YugabyteBugs.bug14330; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLDataType; +import sqlancer.yugabyte.ycql.ast.YCQLBetweenOperation; +import sqlancer.yugabyte.ycql.ast.YCQLBinaryOperation; +import sqlancer.yugabyte.ycql.ast.YCQLColumnReference; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLFunction; +import sqlancer.yugabyte.ycql.ast.YCQLInOperation; +import sqlancer.yugabyte.ycql.ast.YCQLOrderingTerm; +import sqlancer.yugabyte.ycql.ast.YCQLUnaryPostfixOperation; +import sqlancer.yugabyte.ycql.ast.YCQLUnaryPrefixOperation; + +public final class YCQLExpressionGenerator extends UntypedExpressionGenerator { + + private final YCQLGlobalState globalState; + + public YCQLExpressionGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, FUNC, BETWEEN, IN + } + + @Override + protected YCQLExpression generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + YCQLAggregateFunction aggregate = YCQLAggregateFunction.getRandom(); + allowAggregates = false; + return new YCQLFunction<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); + } + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case BINARY_COMPARISON: + Operator op = YCQLBinaryComparisonOperator.getRandom(); + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = YCQLBinaryLogicalOperator.getRandom(); + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + YCQLBinaryArithmeticOperator.getRandom()); + case FUNC: + DBFunction func = DBFunction.getRandom(); + return new YCQLFunction(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new YCQLBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new YCQLInOperation(generateExpression(depth + 1), + generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + default: + throw new AssertionError(expr); + } + } + + @Override + protected YCQLExpression generateColumn() { + YCQLColumn column = Randomly.fromList(columns); + return new YCQLColumnReference(column); + } + + @Override + public YCQLExpression generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + if (bug14330) { + throw new IgnoreMeException(); + } + + return YCQLConstant.createNullConstant(); + } + YCQLDataType type = YCQLDataType.getRandom(); + switch (type) { + case INT: + return YCQLConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DATE: + return YCQLConstant.createDateConstant(globalState.getRandomly().getInteger()); + case TIMESTAMP: + return YCQLConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case VARCHAR: + return YCQLConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOLEAN: + return YCQLConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return YCQLConstant.createFloatConstant(globalState.getRandomly().getDouble()); + default: + throw new AssertionError(); + } + } + + @Override + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (YCQLExpression curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new YCQLOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + }; + + public enum YCQLAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), SUM(1); + + private final int nrArgs; + + YCQLAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static YCQLAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } + + public enum DBFunction { + // YCQL functions + BLOB(1), // + TIMEUUID(1), // + DATE(0), // + TIME(0), // + TIMESTAMP(0), // + BIGINT(1), // + UUID(0); // + // // extras + // PARTITION_HASH(2), // + // WRITETIME(1), // + // TTL(1); // + + private final int nrArgs; + private final boolean isVariadic; + + DBFunction(int nrArgs) { + this(nrArgs, false); + } + + DBFunction(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static DBFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + + } + + public enum YCQLUnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private final String textRepr; + + YCQLUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static YCQLUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLUnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private final String textRepr; + + YCQLUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static YCQLUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLBinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLBinaryArithmeticOperator implements Operator { + ADD("+"), SUB("-"), MULT("*"), DIV("/"); + + private String textRepr; + + YCQLBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum YCQLBinaryComparisonOperator implements Operator { + + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); + + private final String textRepr; + + YCQLBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public YCQLFunction generateArgsForAggregate(YCQLAggregateFunction aggregateFunction) { + return new YCQLFunction(generateExpressions(aggregateFunction.getNrArgs()), + aggregateFunction); + } + + public YCQLExpression generateAggregate() { + YCQLAggregateFunction aggrFunc = YCQLAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + @Override + public YCQLExpression negatePredicate(YCQLExpression predicate) { + return new YCQLUnaryPrefixOperation(predicate, YCQLUnaryPrefixOperator.NOT); + } + + @Override + public YCQLExpression isNull(YCQLExpression expr) { + return new YCQLUnaryPostfixOperation(expr, YCQLUnaryPostfixOperator.IS_NULL); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java new file mode 100644 index 000000000..dd93b38d0 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.yugabyte.ycql.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; + +public class YCQLIndexGenerator extends AbstractIndexGenerator { + + private final YCQLGlobalState globalState; + + public YCQLIndexGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + return new YCQLIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { + errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + } + appendCreateIndex(unique); + sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); + sb.append(" ON "); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + appendIndexColumnList(table.getRandomNonEmptyColumnSubset(), false); + if (Randomly.getBoolean()) { + YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(); + appendWhereClause(YCQLToStringVisitor.asString(expr)); + } + errors.add("Query timed out after PT2S"); + errors.add("Invalid SQL Statement"); + errors.add("Invalid CQL Statement"); + errors.add( + "Invalid Table Definition. Transactions cannot be enabled in an index of a table without transactions enabled."); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java new file mode 100644 index 000000000..167f5d237 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -0,0 +1,55 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; + +public class YCQLInsertGenerator extends AbstractInsertGenerator { + + private final YCQLGlobalState globalState; + + public YCQLInsertGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + return new YCQLInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getColumns(); + buildInsertInto(globalState.getDatabaseName() + "." + table.getName(), columns); + + errors.add("Invalid Arguments"); + errors.add("Null Argument for Primary Key"); + + YCQLErrors.addExpressionErrors(errors); + } + + @Override + protected void insertColumns(List columns) { + sb.append("("); + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + insertValue(columns.get(nrColumn)); + } + sb.append(")"); + } + + @Override + protected void insertValue(YCQLColumn columnYCQL) { + // TODO: select a more meaningful value + sb.append(YCQLToStringVisitor.asString(new YCQLExpressionGenerator(globalState).generateConstant())); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java new file mode 100644 index 000000000..f7692ebb0 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java @@ -0,0 +1,53 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTables; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLSelect; +import sqlancer.yugabyte.ycql.ast.YCQLTableReference; + +public final class YCQLRandomQuerySynthesizer { + + private YCQLRandomQuerySynthesizer() { + } + + public static YCQLSelect generateSelect(YCQLGlobalState globalState, int nrColumns) { + YCQLTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(targetTables.getColumns()); + YCQLSelect select = new YCQLSelect(); + List columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + YCQLExpression expression = gen.generateExpression(); + columns.add(expression); + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + Optional table = tables.stream().map(t -> new YCQLTableReference(t)).findFirst(); + select.setFromList(table.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(Randomly.nonEmptySubset(select.getFetchColumns())); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YCQLConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(YCQLConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + return select; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java new file mode 100644 index 000000000..c57f94287 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java @@ -0,0 +1,54 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; + +public class YCQLTableGenerator extends AbstractTableGenerator { + + private YCQLGlobalState globalState; + + public YCQLTableGenerator() { + this.canAffectSchema = true; + } + + public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { + String tableName = globalState.getSchema().getFreeTableName(); + appendCreateTable(tableName, Randomly.getBoolean()); + List columns = getNewColumns(); + sb.append("("); + appendColumnDefinitionList(columns); + errors.add("Query timed out after PT2S"); + errors.add("Invalid type for index"); + List primaryKeyColumns = Randomly.nonEmptySubset(columns); + sb.append(", PRIMARY KEY("); + sb.append(primaryKeyColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(")"); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + YCQLCompositeDataType columnType = YCQLCompositeDataType.getRandom(); + columns.add(new YCQLColumn(columnName, columnType, false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java new file mode 100644 index 000000000..eee49bee4 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -0,0 +1,59 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; + +public final class YCQLUpdateGenerator extends AbstractUpdateGenerator { + + private final YCQLGlobalState globalState; + private YCQLExpressionGenerator gen; + + private YCQLUpdateGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + return new YCQLUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + errors.add("Invalid Arguments"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Datatype Mismatch"); + errors.add("Null Argument for Primary Key"); + errors.add("Missing Argument for Primary Key"); + + YCQLErrors.addExpressionErrors(errors); + } + + @Override + protected void updateValue(YCQLColumn column) { + YCQLExpression expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + YCQLErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(YCQLToStringVisitor.asString(expr)); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java new file mode 100644 index 000000000..7c21b7db8 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java @@ -0,0 +1,73 @@ +package sqlancer.yugabyte.ycql.test; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.gen.YCQLRandomQuerySynthesizer; + +public class YCQLFuzzer implements TestOracle { + private final YCQLProvider.YCQLGlobalState globalState; + private final List testQueries; + private final ExpectedErrors errors = new ExpectedErrors(); + + public YCQLFuzzer(YCQLProvider.YCQLGlobalState globalState) { + this.globalState = globalState; + + errors.add("Query timed out after PT2S"); + errors.add("Datatype Mismatch"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Invalid Arguments"); + errors.add("Invalid Function Call"); + + testQueries = new ArrayList<>(); + + testQueries.add(new SelectQuery()); + testQueries.add(new ActionQuery(YCQLProvider.Action.UPDATE)); + testQueries.add(new ActionQuery(YCQLProvider.Action.DELETE)); + testQueries.add(new ActionQuery(YCQLProvider.Action.INSERT)); + } + + @Override + public void check() throws Exception { + Query s = testQueries.get(globalState.getRandomly().getInteger(0, testQueries.size())); + globalState.executeStatement(s.getQuery(globalState, errors)); + globalState.getManager().incrementSelectQueryCount(); + } + + private static class Query { + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + throw new IllegalAccessException("Should be implemented"); + }; + } + + private static class ActionQuery extends Query { + private final YCQLProvider.Action action; + + ActionQuery(YCQLProvider.Action action) { + this.action = action; + } + + @Override + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + return action.getQuery(state); + } + } + + private static class SelectQuery extends Query { + + @Override + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + return new SQLQueryAdapter( + YCQLToStringVisitor.asString( + YCQLRandomQuerySynthesizer.generateSelect(state, Randomly.smallNumber() + 1)) + ";", + errors); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java b/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java new file mode 100644 index 000000000..3a8b8946d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql; + +import java.util.Optional; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public final class YSQLCompoundDataType { + + private final YSQLDataType dataType; + private final YSQLCompoundDataType elemType; + private final Integer size; + + private YSQLCompoundDataType(YSQLDataType dataType, YSQLCompoundDataType elemType, Integer size) { + this.dataType = dataType; + this.elemType = elemType; + this.size = size; + } + + public static YSQLCompoundDataType create(YSQLDataType type, int size) { + return new YSQLCompoundDataType(type, null, size); + } + + public static YSQLCompoundDataType create(YSQLDataType type) { + return new YSQLCompoundDataType(type, null, null); + } + + public YSQLDataType getDataType() { + return dataType; + } + + public YSQLCompoundDataType getElemType() { + if (elemType == null) { + throw new AssertionError(); + } + return elemType; + } + + public Optional getSize() { + if (size == null) { + return Optional.empty(); + } else { + return Optional.of(size); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLErrors.java b/src/sqlancer/yugabyte/ysql/YSQLErrors.java new file mode 100644 index 000000000..de692b352 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLErrors.java @@ -0,0 +1,234 @@ +package sqlancer.yugabyte.ysql; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class YSQLErrors { + + private YSQLErrors() { + } + + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("An I/O error occurred while sending to the backend"); + errors.add("Conflicts with committed transaction"); + errors.add("cannot be changed"); + errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); + + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); + errors.add("but it cannot be referenced from this part of the query"); + errors.add("missing FROM-clause entry for table"); + + errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + + return errors; + } + + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); + errors.add("INDEX on column of type 'INET' not yet supported"); + errors.add("INDEX on column of type 'VARBIT' not yet supported"); + errors.add("INDEX on column of type 'INT4RANGE' not yet supported"); + errors.add("is not commutative"); // exclude + errors.add("cannot be changed"); + errors.add("operator requires run-time type coercion"); // exclude + + return errors; + } + + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("syntax error at or near \"(\""); + errors.add("does not exist"); + errors.add("is not unique"); + errors.add("cannot be changed"); + errors.add("invalid reference to FROM-clause entry for table"); + + errors.add("Invalid column number"); + errors.add("specified more than once"); + errors.add("You might need to add explicit type casts"); + errors.add("invalid regular expression"); + errors.add("could not determine which collation to use"); + errors.add("invalid input syntax for integer"); + errors.add("invalid regular expression"); + errors.add("operator does not exist"); + errors.add("quantifier operand invalid"); + errors.add("collation mismatch"); + errors.add("collations are not supported"); + errors.add("operator is not unique"); + errors.add("is not a valid binary digit"); + errors.add("invalid hexadecimal digit"); + errors.add("invalid hexadecimal data: odd number of digits"); + errors.add("zero raised to a negative power is undefined"); + errors.add("cannot convert infinity to numeric"); + errors.add("division by zero"); + errors.add("invalid input syntax for type money"); + errors.add("invalid input syntax for type"); + errors.add("cannot cast type"); + errors.add("value overflows numeric format"); + errors.add("is of type boolean but expression is of type text"); + errors.add("a negative number raised to a non-integer power yields a complex result"); + errors.add("could not determine polymorphic type because input has type unknown"); + + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("multiple decimal points"); + errors.add("and decimal point together"); + errors.add("multiple decimal points"); + errors.add("cannot use \"S\" twice"); + errors.add("must be ahead of \"PR\""); + errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); + errors.add("cannot use \"S\" and \"SG\" together"); + errors.add("cannot use \"S\" and \"MI\" together"); + errors.add("cannot use \"S\" and \"PL\" together"); + errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); + errors.add("is not a number"); + + return errors; + } + + public static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.addAll(getToCharFunctionErrors()); + } + + public static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot XOR bit strings of different sizes"); + errors.add("cannot AND bit strings of different sizes"); + errors.add("cannot OR bit strings of different sizes"); + errors.add("must be type boolean, not type text"); + + return errors; + } + + public static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.addAll(getBitStringOperationErrors()); + } + + public static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("out of valid range"); // get_bit/get_byte + errors.add("cannot take logarithm of a negative number"); + errors.add("cannot take logarithm of zero"); + errors.add("requested character too large for encoding"); // chr + errors.add("null character not permitted"); // chr + errors.add("requested character not valid for encoding"); // chr + errors.add("requested length too large"); // repeat + errors.add("invalid memory alloc request size"); // repeat + errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + + return errors; + } + + public static void addFunctionErrors(ExpectedErrors errors) { + errors.addAll(getFunctionErrors()); + } + + public static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("is not a valid hexadecimal digit"); + + return errors; + } + + public static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); + } + + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("range lower bound must be less than or equal to range upper bound"); + errors.add("result of range difference would not be contiguous"); + errors.add("out of range"); + errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + + return errors; + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.add("value too long for type character"); + errors.add("not found in view targetlist"); + } + + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in GROUP BY"); + + return errors; + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); + } + + public static List getViewErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("already exists"); + errors.add("cannot drop columns from view"); + errors.add("non-integer constant in ORDER BY"); // TODO + errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO + errors.add("cannot change data type of view column"); + errors.add("specified more than once"); // TODO + errors.add("materialized views must not use temporary tables or views"); + errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); + errors.add("is not a view"); + errors.add("non-integer constant in DISTINCT ON"); + errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + + return errors; + } + + public static void addViewErrors(ExpectedErrors errors) { + errors.addAll(getViewErrors()); + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java new file mode 100644 index 000000000..cf175ca39 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java @@ -0,0 +1,152 @@ +package sqlancer.yugabyte.ysql; + +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public final class YSQLExpectedValueVisitor implements YSQLVisitor { + + private static final int NR_TABS = 0; + private final StringBuilder sb = new StringBuilder(); + + private void print(YSQLExpression expr) { + YSQLToStringVisitor v = new YSQLToStringVisitor(); + v.visit(expr); + sb.append("\t".repeat(NR_TABS)); + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(YSQLConstant constant) { + print(constant); + } + + @Override + public void visit(YSQLPostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(YSQLColumnValue c) { + print(c); + } + + @Override + public void visit(YSQLPrefixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(YSQLSelect op) { + visit(op.getWhereClause()); + } + + @Override + public void visit(YSQLOrderByTerm op) { + + } + + @Override + public void visit(YSQLFunction f) { + print(f); + for (int i = 0; i < f.getArguments().length; i++) { + visit(f.getArguments()[i]); + } + } + + @Override + public void visit(YSQLCastOperation cast) { + print(cast); + visit(cast.getExpression()); + } + + @Override + public void visit(YSQLBetweenOperation op) { + print(op); + visit(op.getExpr()); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(YSQLInOperation op) { + print(op); + visit(op.getExpr()); + for (YSQLExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(YSQLPostfixText op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(YSQLAggregate op) { + print(op); + for (YSQLExpression expr : op.getArgs()) { + visit(expr); + } + } + + @Override + public void visit(YSQLSimilarTo op) { + print(op); + visit(op.getString()); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + } + + @Override + public void visit(YSQLPOSIXRegularExpression op) { + print(op); + visit(op.getString()); + visit(op.getRegex()); + } + + @Override + public void visit(YSQLFromTable from) { + print(from); + } + + @Override + public void visit(YSQLSubquery subquery) { + print(subquery); + } + + @Override + public void visit(YSQLBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public String get() { + return sb.toString(); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java b/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java new file mode 100644 index 000000000..093f06aed --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java @@ -0,0 +1,127 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; + +public class YSQLGlobalState extends SQLGlobalState { + + public static final char IMMUTABLE = 'i'; + public static final char STABLE = 's'; + public static final char VOLATILE = 'v'; + // store and allow filtering by function volatility classifications + private final Map functionsAndTypes = new HashMap<>(); + private List operators = Collections.emptyList(); + private List collates = Collections.emptyList(); + private List opClasses = Collections.emptyList(); + private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + + @Override + public void setConnection(SQLConnection con) { + super.setConnection(con); + try { + this.opClasses = getOpclasses(getConnection()); + this.operators = getOperators(getConnection()); + this.collates = getCollnames(getConnection()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + @Override + public YSQLSchema readSchema() throws SQLException { + return YSQLSchema.fromConnection(getConnection(), getDatabaseName()); + } + + private List getCollnames(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + private List getOpclasses(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select opcname FROM pg_opclass;")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + private List getOperators(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + public List getOperators() { + return operators; + } + + public String getRandomOperator() { + return Randomly.fromList(operators); + } + + public List getCollates() { + return collates; + } + + public String getRandomCollate() { + return Randomly.fromList(collates); + } + + public List getOpClasses() { + return opClasses; + } + + public String getRandomOpclass() { + return Randomly.fromList(opClasses); + } + + public void addFunctionAndType(String functionName, Character functionType) { + this.functionsAndTypes.put(functionName, functionType); + } + + public Map getFunctionsAndTypes() { + return this.functionsAndTypes; + } + + public void setDefaultAllowedFunctionTypes() { + this.allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + } + + public List getAllowedFunctionTypes() { + return this.allowedFunctionTypes; + } + + public void setAllowedFunctionTypes(List types) { + this.allowedFunctionTypes = types; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLOptions.java b/src/sqlancer/yugabyte/ysql/YSQLOptions.java new file mode 100644 index 000000000..71b886ba5 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLOptions.java @@ -0,0 +1,35 @@ +package sqlancer.yugabyte.ysql; + +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; + +@Parameters(separators = "=", commandDescription = "YSQL (default port: " + YSQLOptions.DEFAULT_PORT + + ", default host: " + YSQLOptions.DEFAULT_HOST) +public class YSQLOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 5433; + + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) + public boolean allowBulkInsert; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for YSQL") + public List oracle = Arrays.asList(YSQLOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) + public boolean testCollations = true; + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the YSQL server", arity = 1) + public String connectionURL = String.format("jdbc:yugabytedb://%s:%d/yugabyte", YSQLOptions.DEFAULT_HOST, + YSQLOptions.DEFAULT_PORT); + + @Override + public List getTestOracleFactory() { + return oracle; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java b/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java new file mode 100644 index 000000000..b85f80593 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java @@ -0,0 +1,72 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; +import sqlancer.yugabyte.ysql.oracle.YSQLCatalog; +import sqlancer.yugabyte.ysql.oracle.YSQLFuzzer; +import sqlancer.yugabyte.ysql.oracle.YSQLPivotedQuerySynthesisOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPAggregateOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPHavingOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPWhereOracle; + +public enum YSQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLFuzzer(globalState); + } + }, + CATALOG { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLCatalog(globalState); + } + }, + NOREC { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) + .with(YSQLErrors.getCommonFetchErrors()).with("canceling statement due to statement timeout") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + HAVING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new YSQLTLPWhereOracle(globalState)); + oracles.add(new YSQLTLPHavingOracle(globalState)); + oracles.add(new YSQLTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java new file mode 100644 index 000000000..efcc7ec22 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -0,0 +1,373 @@ +package sqlancer.yugabyte.ysql; + +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.gen.YSQLAlterTableGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLAnalyzeGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLCommentGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDeleteGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDiscardGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDropIndexGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLIndexGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLInsertGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLNotifyGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLSequenceGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLSetGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTableGroupGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTransactionGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTruncateGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLUpdateGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLVacuumGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLViewGenerator; + +@AutoService(DatabaseProvider.class) +public class YSQLProvider extends SQLProviderAdapter { + + // TODO Due to yugabyte problems with parallel DDL we need this lock object + public static final Object DDL_LOCK = new Object(); + /** + * Generate only data types and expressions that are understood by PQS. + */ + public static boolean generateOnlyKnown; + protected String entryURL; + protected String username; + protected String password; + protected String entryPath; + protected String host; + protected int port; + protected String testURL; + protected String databaseName; + protected String createDatabaseCommand; + + public YSQLProvider() { + super(YSQLGlobalState.class, YSQLOptions.class); + } + + protected YSQLProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + public static int mapActions(YSQLGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + switch (a) { + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 3); + break; + case DISCARD: + case DROP_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case COMMIT: + nrPerformed = r.getInteger(0, 0); + break; + case ALTER_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case RESET: + nrPerformed = r.getInteger(0, 3); + break; + case ANALYZE: + nrPerformed = r.getInteger(0, 3); + break; + case TABLEGROUP: + nrPerformed = r.getInteger(0, 3); + break; + case DELETE: + case RESET_ROLE: + case VACUUM: + case SET_CONSTRAINTS: + case SET: + case COMMENT_ON: + case NOTIFY: + case LISTEN: + case UNLISTEN: + case CREATE_SEQUENCE: + case TRUNCATE: + nrPerformed = r.getInteger(0, 2); + break; + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + + } + + @Override + public void generateDatabase(YSQLGlobalState globalState) throws Exception { + readFunctions(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + } + + @Override + public SQLConnection createDatabase(YSQLGlobalState globalState) throws SQLException { + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + entryPath = "/yugabyte"; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; + String entryDatabaseName = entryPath.substring(1); + databaseName = globalState.getDatabaseName(); + + if (host == null) { + host = YSQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = YSQLOptions.DEFAULT_PORT; + } + + try { + URI uri = new URI(entryURL); + String userInfoURI = uri.getUserInfo(); + String pathURI = uri.getPath(); + if (userInfoURI != null) { + // username and password specified in URL take precedence + if (userInfoURI.contains(":")) { + String[] userInfo = userInfoURI.split(":", 2); + username = userInfo[0]; + password = userInfo[1]; + } else { + username = userInfoURI; + password = null; + } + int userInfoIndex = entryURL.indexOf(userInfoURI); + String preUserInfo = entryURL.substring(0, userInfoIndex); + String postUserInfo = entryURL.substring(userInfoIndex + userInfoURI.length() + 1); + entryURL = preUserInfo + postUserInfo; + } + if (pathURI != null) { + entryPath = pathURI; + } + if (host == null) { + host = uri.getHost(); + } + if (port == MainOptions.NO_SET_PORT) { + port = uri.getPort(); + } + entryURL = String.format("jdbc:yugabytedb://%s:%d/%s", host, port, entryDatabaseName); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + + createDatabaseSync(globalState, entryDatabaseName); + + int databaseIndex = entryURL.indexOf("/" + entryDatabaseName) + 1; + String preDatabaseName = entryURL.substring(0, databaseIndex); + String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); + testURL = preDatabaseName + databaseName + postDatabaseName; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + + return new SQLConnection(createConnectionSafely(testURL, username, password)); + } + + @Override + public String getDBMSName() { + return "ysql"; + } + + // for some reason yugabyte unable to create few databases simultaneously + private void createDatabaseSync(YSQLGlobalState globalState, String entryDatabaseName) throws SQLException { + synchronized (DDL_LOCK) { + exceptionLessSleep(5000); + + Connection con = createConnectionSafely(entryURL, username, password); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)"); + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)"); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + } + } + + private Connection createConnectionSafely(String entryURL, String user, String password) { + Connection con = null; + IllegalStateException lastException = new IllegalStateException("Empty exception"); + long endTime = System.currentTimeMillis() + 30000; + while (System.currentTimeMillis() < endTime) { + try { + con = DriverManager.getConnection(entryURL, user, password); + break; + } catch (SQLException throwables) { + lastException = new IllegalStateException(throwables); + } + } + + if (con == null) { + throw lastException; + } + + return con; + } + + protected void readFunctions(YSQLGlobalState globalState) throws SQLException { + SQLQueryAdapter query = new SQLQueryAdapter("SELECT proname, provolatile FROM pg_proc;"); + SQLancerResultSet rs = query.executeAndGet(globalState); + while (rs.next()) { + String functionName = rs.getString(1); + Character functionType = rs.getString(2).charAt(0); + globalState.addFunctionAndType(functionName, functionType); + } + } + + protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { + synchronized (DDL_LOCK) { + boolean prevCreationFailed = false; // small optimization - wait only after failed requests + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + if (!prevCreationFailed) { + exceptionLessSleep(5000); + } + + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = YSQLTableGenerator.generate(tableName, generateOnlyKnown, + globalState); + globalState.executeStatement(createTable); + prevCreationFailed = false; + } catch (IgnoreMeException e) { + prevCreationFailed = true; + } + } + } + } + + private void exceptionLessSleep(long timeout) { + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + throw new AssertionError(); + } + } + + protected void prepareTables(YSQLGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + YSQLProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 15000;\n")); + } + + private String getCreateDatabaseCommand(YSQLGlobalState state) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE DATABASE ").append(databaseName).append(" "); + if (Randomly.getBoolean() && state.getDbmsSpecificOptions().testCollations) { + sb.append("WITH "); + if (Randomly.getBoolean()) { + sb.append("ENCODING '"); + sb.append(Randomly.fromOptions("utf8")); + sb.append("' "); + } + + if (Randomly.getBoolean()) { + // if (YugabyteBugs.bug11357) { + // throw new IgnoreMeException(); + // } + + sb.append("COLOCATED = true "); + } + + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } + } + sb.append(" TEMPLATE template0"); + + } + return sb.toString(); + } + + public enum Action implements AbstractAction { + ANALYZE(YSQLAnalyzeGenerator::create), // + ALTER_TABLE(g -> YSQLAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g)), // + COMMIT(g -> { + SQLQueryAdapter query; + if (Randomly.getBoolean()) { + query = new SQLQueryAdapter("COMMIT", true); + } else if (Randomly.getBoolean()) { + query = YSQLTransactionGenerator.executeBegin(); + } else { + query = new SQLQueryAdapter("ROLLBACK", true); + } + return query; + }), // + DELETE(YSQLDeleteGenerator::create), // + DISCARD(YSQLDiscardGenerator::create), // + DROP_INDEX(YSQLDropIndexGenerator::create), // + CREATE_INDEX(YSQLIndexGenerator::generate), // + INSERT(YSQLInsertGenerator::insert), // + UPDATE(YSQLUpdateGenerator::create), // + TRUNCATE(YSQLTruncateGenerator::create), // + TABLEGROUP(YSQLTableGroupGenerator::create), // + VACUUM(YSQLVacuumGenerator::create), // + SET(YSQLSetGenerator::create), // TODO insert yugabyte sets + SET_CONSTRAINTS((g) -> { + String sb = "SET CONSTRAINTS ALL " + Randomly.fromOptions("DEFERRED", "IMMEDIATE"); + return new SQLQueryAdapter(sb); + }), // + RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // + COMMENT_ON(YSQLCommentGenerator::generate), // + RESET((g) -> new SQLQueryAdapter("RESET ALL") /* + * https://www.postgres.org/docs/devel/sql-reset.html TODO: also + * configuration parameter + */), // + NOTIFY(YSQLNotifyGenerator::createNotify), // + LISTEN((g) -> YSQLNotifyGenerator.createListen()), // + UNLISTEN((g) -> YSQLNotifyGenerator.createUnlisten()), // + CREATE_SEQUENCE(YSQLSequenceGenerator::createSequence), // + CREATE_VIEW(YSQLViewGenerator::create); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLSchema.java b/src/sqlancer/yugabyte/ysql/YSQLSchema.java new file mode 100644 index 000000000..c75322af9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLSchema.java @@ -0,0 +1,341 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.postgresql.util.PSQLException; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; + +public class YSQLSchema extends AbstractSchema { + + private final String databaseName; + + public YSQLSchema(List databaseTables, String databaseName) { + super(databaseTables); + this.databaseName = databaseName; + } + + public static YSQLDataType getColumnType(String typeString) { + switch (typeString) { + case "smallint": + case "integer": + case "bigint": + return YSQLDataType.INT; + case "boolean": + return YSQLDataType.BOOLEAN; + case "text": + case "character": + case "character varying": + case "name": + return YSQLDataType.TEXT; + case "numeric": + return YSQLDataType.DECIMAL; + case "double precision": + return YSQLDataType.FLOAT; + case "real": + return YSQLDataType.REAL; + case "int4range": + return YSQLDataType.RANGE; + case "money": + return YSQLDataType.MONEY; + case "bytea": + return YSQLDataType.BYTEA; + case "bit": + case "bit varying": + return YSQLDataType.BIT; + case "inet": + return YSQLDataType.INET; + default: + throw new AssertionError(typeString); + } + } + + public static YSQLSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + try { + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + String tableTypeSchema = rs.getString("table_schema"); + boolean isInsertable = rs.getBoolean("is_insertable_into"); + // TODO: also check insertable + // TODO: insert into view? + boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || + // tableTypeStr.contains("LOCAL TEMPORARY") && + // !isInsertable; + YSQLTable.TableType tableType = getTableType(tableTypeSchema); + List databaseColumns = getTableColumns(con, tableName); + List indexes = getIndexes(con, tableName); + List statistics = getStatistics(con); + YSQLTable t = new YSQLTable(tableName, databaseColumns, indexes, tableType, statistics, isView, + isInsertable); + for (YSQLColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + return new YSQLSchema(databaseTables, databaseName); + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + } + + protected static List getStatistics(SQLConnection con) throws SQLException { + List statistics = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext ORDER BY stxname;")) { + while (rs.next()) { + statistics.add(new YSQLStatisticsObject(rs.getString("stxname"))); + } + } + } + return statistics; + } + + protected static YSQLTable.TableType getTableType(String tableTypeStr) throws AssertionError { + YSQLTable.TableType tableType; + if (tableTypeStr.contentEquals("public")) { + tableType = YSQLTable.TableType.STANDARD; + } else if (tableTypeStr.startsWith("pg_temp")) { + tableType = YSQLTable.TableType.TEMPORARY; + } else { + throw new AssertionError(tableTypeStr); + } + return tableType; + } + + protected static List getIndexes(SQLConnection con, String tableName) throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String + .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' ORDER BY indexname;", tableName))) { + while (rs.next()) { + String indexName = rs.getString("indexname"); + if (DBMSCommon.matchesIndexName(indexName)) { + indexes.add(YSQLIndex.create(indexName)); + } + } + } + } + return indexes; + } + + protected static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("select column_name, data_type from INFORMATION_SCHEMA.COLUMNS where table_name = '" + + tableName + "' ORDER BY column_name")) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + YSQLColumn c = new YSQLColumn(columnName, getColumnType(dataType)); + columns.add(c); + } + } + } + return columns; + } + + public boolean getDatabaseIsColocated(SQLConnection con) { + try (Statement s = con.createStatement(); ResultSet rs = s.executeQuery("SELECT yb_is_database_colocated();")) { + rs.next(); + String result = rs.getString(1); + // The query will result in a 'f' for a non-colocated database + return !"f".equals(result); + + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + public YSQLTables getRandomTableNonEmptyTables() { + return new YSQLTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public String getDatabaseName() { + return databaseName; + } + + public enum YSQLDataType { + // TODO: 23.02.2022 Planned types + // SMALLINT, INT, BIGINT, NUMERIC, DECIMAL, REAL, DOUBLE_PRECISION, VARCHAR, CHAR, TEXT, DATE, TIME, + // TIMESTAMP, TIMESTAMPZ, INTERVAL, INTEGER_ARR + INT, BOOLEAN, BYTEA, TEXT, DECIMAL, FLOAT, REAL, RANGE, MONEY, BIT, INET; + + public static YSQLDataType getRandomType() { + List dataTypes = new ArrayList<>(Arrays.asList(values())); + if (YSQLProvider.generateOnlyKnown) { + dataTypes.remove(YSQLDataType.DECIMAL); + dataTypes.remove(YSQLDataType.FLOAT); + dataTypes.remove(YSQLDataType.REAL); + dataTypes.remove(YSQLDataType.INET); + dataTypes.remove(YSQLDataType.RANGE); + dataTypes.remove(YSQLDataType.MONEY); + dataTypes.remove(YSQLDataType.BIT); + } + return Randomly.fromList(dataTypes); + } + } + + public static class YSQLColumn extends AbstractTableColumn { + + public YSQLColumn(String name, YSQLDataType columnType) { + super(name, null, columnType); + } + + public static YSQLColumn createDummy(String name) { + return new YSQLColumn(name, YSQLDataType.INT); + } + + } + + public static class YSQLTables extends AbstractTables { + + public YSQLTables(List tables) { + super(tables); + } + + public YSQLRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + + // c.getName() + ")") + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new AssertionError("could not find random row! " + randomRow + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + YSQLColumn column = getColumns().get(i); + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + YSQLConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + constant = YSQLConstant.createNullConstant(); + } else { + switch (column.getType()) { + case INT: + constant = YSQLConstant.createIntConstant(randomRowValues.getLong(columnIndex)); + break; + case BOOLEAN: + constant = YSQLConstant.createBooleanConstant(randomRowValues.getBoolean(columnIndex)); + break; + case TEXT: + constant = YSQLConstant.createTextConstant(randomRowValues.getString(columnIndex)); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new YSQLRowValue(this, values); + } catch (PSQLException e) { + throw new IgnoreMeException(); + } + + } + + } + + public static class YSQLRowValue extends AbstractRowValue { + + protected YSQLRowValue(YSQLTables tables, Map values) { + super(tables, values); + } + + } + + public static class YSQLTable extends AbstractRelationalTable { + + private final TableType tableType; + private final List statistics; + private final boolean isInsertable; + + public YSQLTable(String tableName, List columns, List indexes, TableType tableType, + List statistics, boolean isView, boolean isInsertable) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + } + + public List getStatistics() { + return statistics; + } + + public TableType getTableType() { + return tableType; + } + + public boolean isInsertable() { + return isInsertable; + } + + public enum TableType { + STANDARD, TEMPORARY + } + + } + + public static final class YSQLStatisticsObject { + private final String name; + + public YSQLStatisticsObject(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + public static final class YSQLIndex extends TableIndex { + + private YSQLIndex(String indexName) { + super(indexName); + } + + public static YSQLIndex create(String indexName) { + return new YSQLIndex(indexName); + } + + @Override + public String getIndexName() { + if (super.getIndexName().contentEquals("PRIMARY")) { + return "`PRIMARY`"; + } else { + return super.getIndexName(); + } + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java new file mode 100644 index 000000000..54e718768 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java @@ -0,0 +1,329 @@ +package sqlancer.yugabyte.ysql; + +import java.util.Optional; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLJoin.YSQLJoinType; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public final class YSQLToStringVisitor extends ToStringVisitor implements YSQLVisitor { + + @Override + public void visitSpecific(YSQLExpression expr) { + YSQLVisitor.super.visit(expr); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(YSQLConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public void visit(YSQLPostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" "); + sb.append(op.getOperatorTextRepresentation()); + } + + @Override + public void visit(YSQLColumnValue c) { + sb.append(c.getColumn().getFullQualifiedName()); + } + + @Override + public void visit(YSQLPrefixOperation op) { + sb.append(op.getTextRepresentation()); + sb.append(" ("); + visit(op.getExpression()); + sb.append(")"); + } + + @Override + public void visit(YSQLSelect s) { + sb.append("SELECT "); + switch (s.getSelectOption()) { + case DISTINCT: + sb.append("DISTINCT "); + if (s.getDistinctOnClause() != null) { + sb.append("ON ("); + visit(s.getDistinctOnClause()); + sb.append(") "); + } + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + if (s.getFetchColumns() == null) { + sb.append("*"); + } else { + visit(s.getFetchColumns()); + } + sb.append(" FROM "); + visit(s.getFromList()); + + for (YSQLJoin j : s.getJoinClauses()) { + sb.append(" "); + switch (j.getType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } + sb.append("JOIN"); + break; + case LEFT: + sb.append("LEFT OUTER JOIN"); + break; + case RIGHT: + sb.append("RIGHT OUTER JOIN"); + break; + case FULL: + sb.append("FULL OUTER JOIN"); + break; + case CROSS: + sb.append("CROSS JOIN"); + break; + default: + throw new AssertionError(j.getType()); + } + sb.append(" "); + visit(j.getTableReference()); + if (j.getType() != YSQLJoinType.CROSS) { + sb.append(" ON "); + visit(j.getOnClause()); + } + } + + if (s.getWhereClause() != null) { + sb.append(" WHERE "); + visit(s.getWhereClause()); + } + if (!s.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(s.getGroupByExpressions()); + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + + } + if (!s.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(s.getOrderByClauses()); + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(YSQLOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder()); + } + + @Override + public void visit(YSQLFunction f) { + sb.append(f.getFunctionName()); + sb.append("("); + int i = 0; + for (YSQLExpression arg : f.getArguments()) { + if (i++ != 0) { + sb.append(", "); + } + visit(arg); + } + sb.append(")"); + } + + @Override + public void visit(YSQLCastOperation cast) { + if (Randomly.getBoolean()) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + appendType(cast); + sb.append(")"); + } else { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::"); + appendType(cast); + } + } + + @Override + public void visit(YSQLBetweenOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(") BETWEEN "); + if (op.isSymmetric()) { + sb.append("SYMMETRIC "); + } + sb.append("("); + visit(op.getLeft()); + sb.append(") AND ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(YSQLInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN ("); + visit(op.getListElements()); + sb.append(")"); + } + + @Override + public void visit(YSQLPostfixText op) { + visit(op.getExpr()); + sb.append(op.getText()); + } + + @Override + public void visit(YSQLAggregate op) { + sb.append(op.getFunction()); + sb.append("("); + visit(op.getArgs()); + sb.append(")"); + } + + @Override + public void visit(YSQLSimilarTo op) { + sb.append("("); + visit(op.getString()); + sb.append(" SIMILAR TO "); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + sb.append(")"); + } + + @Override + public void visit(YSQLPOSIXRegularExpression op) { + visit(op.getString()); + sb.append(op.getOp().getStringRepresentation()); + visit(op.getRegex()); + } + + @Override + public void visit(YSQLFromTable from) { + if (from.isOnly()) { + sb.append("ONLY "); + } + sb.append(from.getTable().getName()); + if (!from.isOnly() && Randomly.getBoolean()) { + sb.append("*"); + } + } + + @Override + public void visit(YSQLSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(YSQLBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + private void appendType(YSQLCastOperation cast) { + YSQLCompoundDataType compoundType = cast.getCompoundType(); + switch (compoundType.getDataType()) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: // TODO support also other int types + sb.append("INT"); + break; + case TEXT: + // TODO: append TEXT, CHAR + sb.append(Randomly.fromOptions("VARCHAR")); + break; + case REAL: + sb.append("REAL"); + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("FLOAT"); + break; + case RANGE: + sb.append("int4range"); + break; + case MONEY: + sb.append("MONEY"); + break; + case INET: + sb.append("INET"); + break; + case BIT: + sb.append("BIT"); + break; + case BYTEA: + sb.append("BYTEA"); + break; + // if (Randomly.getBoolean()) { + // sb.append("("); + // sb.append(Randomly.getNotCachedInteger(1, 100)); + // sb.append(")"); + // } + default: + throw new AssertionError(cast.getType()); + } + Optional size = compoundType.getSize(); + if (size.isPresent()) { + sb.append("("); + sb.append(size.get()); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLVisitor.java new file mode 100644 index 000000000..a73aade4c --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLVisitor.java @@ -0,0 +1,120 @@ +package sqlancer.yugabyte.ysql; + +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public interface YSQLVisitor { + + static String asString(YSQLExpression expr) { + YSQLToStringVisitor visitor = new YSQLToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(YSQLExpression expr) { + YSQLExpectedValueVisitor v = new YSQLExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + + static String getExpressionAsString(YSQLGlobalState globalState, YSQLDataType type, List columns) { + YSQLExpression expression = YSQLExpressionGenerator.generateExpression(globalState, columns, type); + YSQLToStringVisitor visitor = new YSQLToStringVisitor(); + visitor.visit(expression); + return visitor.get(); + } + + void visit(YSQLConstant constant); + + void visit(YSQLPostfixOperation op); + + void visit(YSQLColumnValue c); + + void visit(YSQLPrefixOperation op); + + void visit(YSQLSelect op); + + void visit(YSQLOrderByTerm op); + + void visit(YSQLFunction f); + + void visit(YSQLCastOperation cast); + + void visit(YSQLBetweenOperation op); + + void visit(YSQLInOperation op); + + void visit(YSQLPostfixText op); + + void visit(YSQLAggregate op); + + void visit(YSQLSimilarTo op); + + void visit(YSQLPOSIXRegularExpression op); + + void visit(YSQLFromTable from); + + void visit(YSQLSubquery subquery); + + void visit(YSQLBinaryLogicalOperation op); + + default void visit(YSQLExpression expression) { + if (expression instanceof YSQLConstant) { + visit((YSQLConstant) expression); + } else if (expression instanceof YSQLPostfixOperation) { + visit((YSQLPostfixOperation) expression); + } else if (expression instanceof YSQLColumnValue) { + visit((YSQLColumnValue) expression); + } else if (expression instanceof YSQLPrefixOperation) { + visit((YSQLPrefixOperation) expression); + } else if (expression instanceof YSQLSelect) { + visit((YSQLSelect) expression); + } else if (expression instanceof YSQLOrderByTerm) { + visit((YSQLOrderByTerm) expression); + } else if (expression instanceof YSQLFunction) { + visit((YSQLFunction) expression); + } else if (expression instanceof YSQLCastOperation) { + visit((YSQLCastOperation) expression); + } else if (expression instanceof YSQLBetweenOperation) { + visit((YSQLBetweenOperation) expression); + } else if (expression instanceof YSQLInOperation) { + visit((YSQLInOperation) expression); + } else if (expression instanceof YSQLAggregate) { + visit((YSQLAggregate) expression); + } else if (expression instanceof YSQLPostfixText) { + visit((YSQLPostfixText) expression); + } else if (expression instanceof YSQLSimilarTo) { + visit((YSQLSimilarTo) expression); + } else if (expression instanceof YSQLPOSIXRegularExpression) { + visit((YSQLPOSIXRegularExpression) expression); + } else if (expression instanceof YSQLFromTable) { + visit((YSQLFromTable) expression); + } else if (expression instanceof YSQLSubquery) { + visit((YSQLSubquery) expression); + } else { + throw new AssertionError(expression); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java b/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java new file mode 100644 index 000000000..27daefcf9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java @@ -0,0 +1,58 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; + +/** + * @see Built-in Aggregate Functions + */ +public class YSQLAggregate extends FunctionNode implements YSQLExpression { + + public YSQLAggregate(List args, YSQLAggregateFunction func) { + super(func, args); + } + + public enum YSQLAggregateFunction { + AVG(YSQLDataType.INT, YSQLDataType.FLOAT, YSQLDataType.REAL, YSQLDataType.DECIMAL), BIT_AND(YSQLDataType.INT), + BIT_OR(YSQLDataType.INT), BOOL_AND(YSQLDataType.BOOLEAN), BOOL_OR(YSQLDataType.BOOLEAN), + COUNT(YSQLDataType.INT), EVERY(YSQLDataType.BOOLEAN), MAX, MIN, + // STRING_AGG + SUM(YSQLDataType.INT, YSQLDataType.FLOAT, YSQLDataType.REAL, YSQLDataType.DECIMAL); + + private final YSQLDataType[] supportedReturnTypes; + + YSQLAggregateFunction(YSQLDataType... supportedReturnTypes) { + this.supportedReturnTypes = supportedReturnTypes.clone(); + } + + public static List getAggregates(YSQLDataType type) { + return Arrays.stream(values()).filter(p -> p.supportsReturnType(type)).collect(Collectors.toList()); + } + + public List getTypes(YSQLDataType returnType) { + return Collections.singletonList(returnType); + } + + public boolean supportsReturnType(YSQLDataType returnType) { + return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType) + || supportedReturnTypes.length == 0; + } + + public YSQLDataType getRandomReturnType() { + if (supportedReturnTypes.length == 0) { + return Randomly.fromOptions(YSQLDataType.getRandomType()); + } else { + return Randomly.fromOptions(supportedReturnTypes); + } + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java b/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java new file mode 100644 index 000000000..96e432370 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java @@ -0,0 +1,35 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.common.visitor.UnaryOperation; + +public class YSQLAlias implements UnaryOperation, YSQLExpression { + + private final YSQLExpression expr; + private final String alias; + + public YSQLAlias(YSQLExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + @Override + public YSQLExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return " as " + alias; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java new file mode 100644 index 000000000..4a6c123d0 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java @@ -0,0 +1,63 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public final class YSQLBetweenOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final YSQLExpression left; + private final YSQLExpression right; + private final boolean isSymmetric; + + public YSQLBetweenOperation(YSQLExpression expr, YSQLExpression left, YSQLExpression right, boolean symmetric) { + this.expr = expr; + this.left = left; + this.right = right; + isSymmetric = symmetric; + } + + public YSQLExpression getExpr() { + return expr; + } + + public YSQLExpression getLeft() { + return left; + } + + public YSQLExpression getRight() { + return right; + } + + public boolean isSymmetric() { + return isSymmetric; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLBinaryComparisonOperation leftComparison = new YSQLBinaryComparisonOperation(left, expr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryComparisonOperation rightComparison = new YSQLBinaryComparisonOperation(expr, right, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryLogicalOperation andOperation = new YSQLBinaryLogicalOperation(leftComparison, rightComparison, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.AND); + if (isSymmetric) { + YSQLBinaryComparisonOperation leftComparison2 = new YSQLBinaryComparisonOperation(right, expr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryComparisonOperation rightComparison2 = new YSQLBinaryComparisonOperation(expr, left, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryLogicalOperation andOperation2 = new YSQLBinaryLogicalOperation(leftComparison2, rightComparison2, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.AND); + YSQLBinaryLogicalOperation orOp = new YSQLBinaryLogicalOperation(andOperation, andOperation2, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.OR); + return orOp.getExpectedValue(); + } else { + return andOperation.getExpectedValue(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java new file mode 100644 index 000000000..a4385c86c --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java @@ -0,0 +1,106 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryArithmeticOperation.YSQLBinaryOperator; + +public class YSQLBinaryArithmeticOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryArithmeticOperation(YSQLExpression left, YSQLExpression right, YSQLBinaryOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpected = getLeft().getExpectedValue(); + YSQLConstant rightExpected = getRight().getExpectedValue(); + if (leftExpected == null || rightExpected == null) { + return null; + } + return getOp().apply(leftExpected, rightExpected); + } + + public enum YSQLBinaryOperator implements Operator { + + ADDITION("+") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, Long::sum); + } + + }, + SUBTRACTION("-") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + + } + + }, + MODULO("%") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + + } + }, + EXPONENTIATION("^") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return null; + } + }; + + private final String textRepresentation; + + YSQLBinaryOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + private static YSQLConstant applyBitOperation(YSQLConstant left, YSQLConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + long leftVal = left.cast(YSQLDataType.INT).asInt(); + long rightVal = right.cast(YSQLDataType.INT).asInt(); + long value = op.apply(leftVal, rightVal); + return YSQLConstant.createIntConstant(value); + } + } + + public static YSQLBinaryOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract YSQLConstant apply(YSQLConstant left, YSQLConstant right); + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java new file mode 100644 index 000000000..0d1cb8f2e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java @@ -0,0 +1,46 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryBitOperation.YSQLBinaryBitOperator; + +public class YSQLBinaryBitOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryBitOperation(YSQLBinaryBitOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BIT; + } + + public enum YSQLBinaryBitOperator implements Operator { + CONCATENATION("||"), // + BITWISE_AND("&"), // + BITWISE_OR("|"), // + BITWISE_XOR("#"), // + BITWISE_SHIFT_LEFT("<<"), // + BITWISE_SHIFT_RIGHT(">>"); + + private final String text; + + YSQLBinaryBitOperator(String text) { + this.text = text; + } + + public static YSQLBinaryBitOperator getRandom() { + return Randomly.fromOptions(YSQLBinaryBitOperator.values()); + } + + @Override + public String getTextRepresentation() { + return text; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java new file mode 100644 index 000000000..8ced603ac --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java @@ -0,0 +1,135 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator; + +public class YSQLBinaryComparisonOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryComparisonOperation(YSQLExpression left, YSQLExpression right, YSQLBinaryComparisonOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); + } + + public enum YSQLBinaryComparisonOperator implements Operator { + EQUALS("=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + IS_DISTINCT("IS DISTINCT FROM") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return YSQLConstant + .createBooleanConstant(!IS_NOT_DISTINCT.getExpectedValue(leftVal, rightVal).asBoolean()); + } + }, + IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + if (leftVal.isNull()) { + return YSQLConstant.createBooleanConstant(rightVal.isNull()); + } else if (rightVal.isNull()) { + return YSQLConstant.createFalse(); + } else { + return leftVal.isEquals(rightVal); + } + } + }, + NOT_EQUALS("!=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.isBoolean()) { + return YSQLConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + LESS("<") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan.isBoolean() && !lessThan.asBoolean()) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return YSQLConstant.createFalse(); + } else { + YSQLConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLPrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return YSQLConstant.createTrue(); + } else { + YSQLConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLPrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + + }; + + private final String textRepresentation; + + YSQLBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(YSQLBinaryComparisonOperator.values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal); + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java new file mode 100644 index 000000000..89cce762f --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java @@ -0,0 +1,88 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation.BinaryLogicalOperator; + +public class YSQLBinaryLogicalOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryLogicalOperation(YSQLExpression left, YSQLExpression right, BinaryLogicalOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + + public enum BinaryLogicalOperator implements Operator { + AND { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + YSQLConstant leftBool = left.cast(YSQLDataType.BOOLEAN); + YSQLConstant rightBool = right.cast(YSQLDataType.BOOLEAN); + if (leftBool.isNull()) { + if (rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + if (rightBool.asBoolean()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createFalse(); + } + } + } else if (!leftBool.asBoolean()) { + return YSQLConstant.createFalse(); + } + assert leftBool.asBoolean(); + if (rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(rightBool.isBoolean() && rightBool.asBoolean()); + } + } + }, + OR { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + YSQLConstant leftBool = left.cast(YSQLDataType.BOOLEAN); + YSQLConstant rightBool = right.cast(YSQLDataType.BOOLEAN); + if (leftBool.isBoolean() && leftBool.asBoolean()) { + return YSQLConstant.createTrue(); + } + if (rightBool.isBoolean() && rightBool.asBoolean()) { + return YSQLConstant.createTrue(); + } + if (leftBool.isNull() || rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLConstant.createFalse(); + } + }; + + public static BinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract YSQLConstant apply(YSQLConstant left, YSQLConstant right); + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java new file mode 100644 index 000000000..4bf4a8ea6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java @@ -0,0 +1,71 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLBinaryRangeOperation extends BinaryNode implements YSQLExpression { + + private final String op; + + public YSQLBinaryRangeOperation(YSQLBinaryRangeComparisonOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + public YSQLBinaryRangeOperation(YSQLBinaryRangeOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public String getOperatorRepresentation() { + return op; + } + + public enum YSQLBinaryRangeOperator implements Operator { + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); + + private final String textRepresentation; + + YSQLBinaryRangeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryRangeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum YSQLBinaryRangeComparisonOperator { + CONTAINS_RANGE_OR_ELEMENT("@>"), RANGE_OR_ELEMENT_IS_CONTAINED("<@"), OVERLAP("&&"), STRICT_LEFT_OF("<<"), + STRICT_RIGHT_OF(">>"), NOT_RIGHT_OF("&<"), NOT_LEFT_OF(">&"), ADJACENT("-|-"); + + private final String textRepresentation; + + YSQLBinaryRangeComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryRangeComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java new file mode 100644 index 000000000..cacd8ad76 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLCastOperation implements YSQLExpression { + + private final YSQLExpression expression; + private final YSQLCompoundDataType type; + + public YSQLCastOperation(YSQLExpression expression, YSQLCompoundDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + @Override + public YSQLDataType getExpressionType() { + return type.getDataType(); + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expression.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type.getDataType()); + } + + public YSQLExpression getExpression() { + return expression; + } + + public YSQLDataType getType() { + return type.getDataType(); + } + + public YSQLCompoundDataType getCompoundType() { + return type; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java b/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java new file mode 100644 index 000000000..82b164c54 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java @@ -0,0 +1,33 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLCollate implements YSQLExpression { + + private final YSQLExpression expr; + private final String collate; + + public YSQLCollate(YSQLExpression expr, String collate) { + this.expr = expr; + this.collate = collate; + } + + public String getCollate() { + return collate; + } + + public YSQLExpression getExpr() { + return expr; + } + + @Override + public YSQLDataType getExpressionType() { + return expr.getExpressionType(); + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java b/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java new file mode 100644 index 000000000..243bdaf57 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java @@ -0,0 +1,34 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLColumnValue implements YSQLExpression { + + private final YSQLColumn c; + private final YSQLConstant expectedValue; + + public YSQLColumnValue(YSQLColumn c, YSQLConstant expectedValue) { + this.c = c; + this.expectedValue = expectedValue; + } + + public static YSQLColumnValue create(YSQLColumn c, YSQLConstant expected) { + return new YSQLColumnValue(c, expected); + } + + @Override + public YSQLDataType getExpressionType() { + return c.getType(); + } + + @Override + public YSQLConstant getExpectedValue() { + return expectedValue; + } + + public YSQLColumn getColumn() { + return c; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java new file mode 100644 index 000000000..a74263b22 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java @@ -0,0 +1,37 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.common.ast.BinaryNode; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLConcatOperation extends BinaryNode implements YSQLExpression { + + public YSQLConcatOperation(YSQLExpression left, YSQLExpression right) { + super(left, right); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.TEXT; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + if (leftExpectedValue.isNull() || rightExpectedValue.isNull()) { + return YSQLConstant.createNullConstant(); + } + String leftStr = leftExpectedValue.cast(YSQLDataType.TEXT).getUnquotedTextRepresentation(); + String rightStr = rightExpectedValue.cast(YSQLDataType.TEXT).getUnquotedTextRepresentation(); + return YSQLConstant.createTextConstant(leftStr + rightStr); + } + + @Override + public String getOperatorRepresentation() { + return "||"; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java b/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java new file mode 100644 index 000000000..9fd2eaad6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java @@ -0,0 +1,611 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.math.BigDecimal; + +import sqlancer.IgnoreMeException; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public abstract class YSQLConstant implements YSQLExpression { + + public static YSQLConstant createNullConstant() { + return new YSQLNullConstant(); + } + + public static YSQLConstant createIntConstant(long val) { + return new IntConstant(val); + } + + public static YSQLConstant createBooleanConstant(boolean val) { + return new BooleanConstant(val); + } + + public static YSQLConstant createFalse() { + return createBooleanConstant(false); + } + + public static YSQLConstant createTrue() { + return createBooleanConstant(true); + } + + public static YSQLConstant createTextConstant(String string) { + return new StringConstant(string); + } + + public static YSQLConstant createByteConstant(String string) { + return new ByteConstant(string); + } + + public static YSQLConstant createDecimalConstant(BigDecimal bigDecimal) { + return new DecimalConstant(bigDecimal); + } + + public static YSQLConstant createFloatConstant(float val) { + return new FloatConstant(val); + } + + public static YSQLConstant createDoubleConstant(double val) { + return new DoubleConstant(val); + } + + public static YSQLConstant createRange(long left, boolean leftIsInclusive, long right, boolean rightIsInclusive) { + long realLeft; + long realRight; + if (left > right) { + realRight = left; + realLeft = right; + } else { + realLeft = left; + realRight = right; + } + return new RangeConstant(realLeft, leftIsInclusive, realRight, rightIsInclusive); + } + + public static YSQLExpression createBitConstant(long integer) { + return new BitConstant(integer); + } + + public static YSQLExpression createInetConstant(String val) { + return new InetConstant(val); + } + + public abstract String getTextRepresentation(); + + public abstract String getUnquotedTextRepresentation(); + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isString() { + return false; + } + + @Override + public YSQLConstant getExpectedValue() { + return this; + } + + public boolean isNull() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isBoolean() { + return false; + } + + public abstract YSQLConstant isEquals(YSQLConstant rightVal); + + public boolean isInt() { + return false; + } + + protected abstract YSQLConstant isLessThan(YSQLConstant rightVal); + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract YSQLConstant cast(YSQLDataType type); + + public static class BooleanConstant extends YSQLConstant { + + private final boolean value; + + public BooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return value ? "TRUE" : "FALSE"; + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return YSQLConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value == rightVal.cast(YSQLDataType.BOOLEAN).asBoolean()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isString()) { + return isLessThan(rightVal.cast(YSQLDataType.BOOLEAN)); + } else { + assert rightVal.isBoolean(); + return YSQLConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + switch (type) { + case BOOLEAN: + return this; + case INT: + return YSQLConstant.createIntConstant(value ? 1 : 0); + case TEXT: + return YSQLConstant.createTextConstant(value ? "true" : "false"); + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + } + + public static class YSQLNullConstant extends YSQLConstant { + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + return YSQLConstant.createNullConstant(); + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + return YSQLConstant.createNullConstant(); + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + return YSQLConstant.createNullConstant(); + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + } + + public static class StringConstant extends YSQLConstant { + + protected final String value; + + public StringConstant(String value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'", value.replace("'", "''")); + } + + @Override + public String getUnquotedTextRepresentation() { + return value; + } + + @Override + public String asString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(YSQLDataType.INT).isEquals(rightVal.cast(YSQLDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isEquals(rightVal.cast(YSQLDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(YSQLDataType.INT).isLessThan(rightVal.cast(YSQLDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isLessThan(rightVal.cast(YSQLDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + if (type == YSQLDataType.TEXT) { + return this; + } + String s = value.trim(); + switch (type) { + case BOOLEAN: + try { + return YSQLConstant.createBooleanConstant(Long.parseLong(s) != 0); + } catch (NumberFormatException e) { + } + switch (s.toUpperCase()) { + case "T": + case "TR": + case "TRU": + case "TRUE": + case "1": + case "YES": + case "YE": + case "Y": + case "ON": + return YSQLConstant.createTrue(); + case "F": + case "FA": + case "FAL": + case "FALS": + case "FALSE": + case "N": + case "NO": + case "OF": + case "OFF": + default: + return YSQLConstant.createFalse(); + } + case INT: + try { + return YSQLConstant.createIntConstant(Long.parseLong(s)); + } catch (NumberFormatException e) { + return YSQLConstant.createIntConstant(-1); + } + case TEXT: + return this; + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.TEXT; + } + + } + + public static class IntConstant extends YSQLConstant { + + private final long val; + + public IntConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public long asInt() { + return val; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isEquals(rightVal); + } else if (rightVal.isInt()) { + return YSQLConstant.createBooleanConstant(val == rightVal.asInt()); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(val == rightVal.cast(YSQLDataType.INT).asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public boolean isInt() { + return true; + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return YSQLConstant.createBooleanConstant(val < rightVal.asInt()); + } else if (rightVal.isBoolean()) { + throw new AssertionError(rightVal); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(val < rightVal.cast(YSQLDataType.INT).asInt()); + } else { + throw new IgnoreMeException(); + } + + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + switch (type) { + case BOOLEAN: + return YSQLConstant.createBooleanConstant(val != 0); + case INT: + return this; + case TEXT: + return YSQLConstant.createTextConstant(String.valueOf(val)); + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + } + + public static class ByteConstant extends StringConstant { + + public ByteConstant(String value) { + super(value); + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'::bytea", value.replace("'", "''")); + } + } + + public abstract static class YSQLConstantBase extends YSQLConstant { + + @Override + public String getUnquotedTextRepresentation() { + return null; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + return null; + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + return null; + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + return null; + } + } + + public static class DecimalConstant extends YSQLConstantBase { + + private final BigDecimal val; + + public DecimalConstant(BigDecimal val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.DECIMAL; + } + + } + + public static class InetConstant extends YSQLConstantBase { + + private final String val; + + public InetConstant(String val) { + this.val = "'" + val + "'"; + } + + @Override + public String getTextRepresentation() { + return val; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INET; + } + + } + + public static class FloatConstant extends YSQLConstantBase { + + private final float val; + + public FloatConstant(float val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.FLOAT; + } + + } + + public static class DoubleConstant extends YSQLConstantBase { + + private final double val; + + public DoubleConstant(double val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.FLOAT; + } + + } + + public static class BitConstant extends YSQLConstantBase { + + private final long val; + + public BitConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.format("B'%s'", Long.toBinaryString(val)); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BIT; + } + + } + + public static class RangeConstant extends YSQLConstantBase { + + private final long left; + private final boolean leftIsInclusive; + private final long right; + private final boolean rightIsInclusive; + + public RangeConstant(long left, boolean leftIsInclusive, long right, boolean rightIsInclusive) { + this.left = left; + this.leftIsInclusive = leftIsInclusive; + this.right = right; + this.rightIsInclusive = rightIsInclusive; + } + + @Override + public String getTextRepresentation() { + StringBuilder sb = new StringBuilder(); + sb.append("'"); + if (leftIsInclusive) { + sb.append("["); + } else { + sb.append("("); + } + sb.append(left); + sb.append(","); + sb.append(right); + if (rightIsInclusive) { + sb.append("]"); + } else { + sb.append(")"); + } + sb.append("'"); + sb.append("::int4range"); + return sb.toString(); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.RANGE; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java new file mode 100644 index 000000000..5f68afd5d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java @@ -0,0 +1,16 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public interface YSQLExpression extends Expression { + + default YSQLDataType getExpressionType() { + return null; + } + + default YSQLConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java b/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java new file mode 100644 index 000000000..54349ebe9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java @@ -0,0 +1,283 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLFunction implements YSQLExpression { + + private final String func; + private final YSQLExpression[] args; + private final YSQLDataType returnType; + private YSQLFunctionWithResult functionWithKnownResult; + + public YSQLFunction(YSQLFunctionWithResult func, YSQLDataType returnType, YSQLExpression... args) { + functionWithKnownResult = func; + this.func = func.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public YSQLFunction(YSQLFunctionWithUnknownResult f, YSQLDataType returnType, YSQLExpression... args) { + this.func = f.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public String getFunctionName() { + return func; + } + + public YSQLExpression[] getArguments() { + return args.clone(); + } + + @Override + public YSQLDataType getExpressionType() { + return returnType; + } + + @Override + public YSQLConstant getExpectedValue() { + if (functionWithKnownResult == null) { + return null; + } + YSQLConstant[] constants = new YSQLConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + if (constants[i] == null) { + return null; + } + } + return functionWithKnownResult.apply(constants, args); + } + + public enum YSQLFunctionWithResult { + ABS(1, "abs") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createIntConstant(Math.abs(evaluatedArgs[0].cast(YSQLDataType.INT).asInt())); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { returnType }; + } + + }, + LOWER(1, "lower") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createTextConstant(text.toLowerCase()); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.TEXT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + + }, + LENGTH(1, "length") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createIntConstant(text.length()); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + }, + UPPER(1, "upper") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createTextConstant(text.toUpperCase()); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.TEXT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + + }, + // NULL_IF(2, "nullif") { + // + // @Override + // public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression[] args) { + // YSQLConstant equals = evaluatedArgs[0].isEquals(evaluatedArgs[1]); + // if (equals.isBoolean() && equals.asBoolean()) { + // return YSQLConstant.createNullConstant(); + // } else { + // // TODO: SELECT (nullif('1', FALSE)); yields '1', but should yield TRUE + // return evaluatedArgs[0]; + // } + // } + // + // @Override + // public boolean supportsReturnType(YSQLDataType type) { + // return true; + // } + // + // @Override + // public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + // return getType(nrArguments, returnType); + // } + // + // @Override + // public boolean checkArguments(YSQLExpression[] constants) { + // for (YSQLExpression e : constants) { + // if (!(e instanceof YSQLNullConstant)) { + // return true; + // } + // } + // return false; + // } + // + // }, + NUM_NONNULLS(1, "num_nonnulls") { + @Override + public YSQLConstant apply(YSQLConstant[] args, YSQLExpression... origArgs) { + int nr = 0; + for (YSQLConstant c : args) { + if (!c.isNull()) { + nr++; + } + } + return YSQLConstant.createIntConstant(nr); + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + NUM_NULLS(1, "num_nulls") { + @Override + public YSQLConstant apply(YSQLConstant[] args, YSQLExpression... origArgs) { + int nr = 0; + for (YSQLConstant c : args) { + if (c.isNull()) { + nr++; + } + } + return YSQLConstant.createIntConstant(nr); + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }; + + final int nrArgs; + private final String functionName; + private final boolean variadic; + + YSQLFunctionWithResult(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + public YSQLDataType[] getRandomTypes(int nr) { + YSQLDataType[] types = new YSQLDataType[nr]; + for (int i = 0; i < types.length; i++) { + types[i] = YSQLDataType.getRandomType(); + } + return types; + } + + /** + * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number + * of arguments is returned. + * + * @return the number of arguments + */ + public int getNrArgs() { + return nrArgs; + } + + public abstract YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args); + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + + public abstract boolean supportsReturnType(YSQLDataType type); + + public abstract YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments); + + public boolean checkArguments(YSQLExpression... constants) { + return true; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java b/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java new file mode 100644 index 000000000..8fbd1aa5d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java @@ -0,0 +1,174 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public enum YSQLFunctionWithUnknownResult { + + ABBREV("abbrev", YSQLDataType.TEXT, YSQLDataType.INET), + BROADCAST("broadcast", YSQLDataType.INET, YSQLDataType.INET), FAMILY("family", YSQLDataType.INT, YSQLDataType.INET), + HOSTMASK("hostmask", YSQLDataType.INET, YSQLDataType.INET), MASKLEN("masklen", YSQLDataType.INT, YSQLDataType.INET), + NETMASK("netmask", YSQLDataType.INET, YSQLDataType.INET), + SET_MASKLEN("set_masklen", YSQLDataType.INET, YSQLDataType.INET, YSQLDataType.INT), + TEXT("text", YSQLDataType.TEXT, YSQLDataType.INET), + INET_SAME_FAMILY("inet_same_family", YSQLDataType.BOOLEAN, YSQLDataType.INET, YSQLDataType.INET), + + // https://www.postgres.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE + // PG_RELOAD_CONF("pg_reload_conf", YSQLDataType.BOOLEAN), // too much output + // PG_ROTATE_LOGFILE("pg_rotate_logfile", YSQLDataType.BOOLEAN), prints warning + + // https://www.postgresql.org/docs/devel/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE + CURRENT_DATABASE("current_database", YSQLDataType.TEXT), // name + // CURRENT_QUERY("current_query", YSQLDataType.TEXT), // can generate false positives + CURRENT_SCHEMA("current_schema", YSQLDataType.TEXT), // name + // CURRENT_SCHEMAS("current_schemas", YSQLDataType.TEXT, YSQLDataType.BOOLEAN), + INET_CLIENT_PORT("inet_client_port", YSQLDataType.INT), INET_SERVER_PORT("inet_server_port", YSQLDataType.INT), + PG_BACKEND_PID("pg_backend_pid", YSQLDataType.INT), PG_CURRENT_LOGFILE("pg_current_logfile", YSQLDataType.TEXT), + // PG_IS_OTHER_TEMP_SCHEMA("pg_is_other_temp_schema", YSQLDataType.BOOLEAN), + // PG_JIT_AVAILABLE("pg_is_other_temp_schema", YSQLDataType.BOOLEAN), + PG_NOTIFICATION_QUEUE_USAGE("pg_notification_queue_usage", YSQLDataType.REAL), + PG_TRIGGER_DEPTH("pg_trigger_depth", YSQLDataType.INT), VERSION("version", YSQLDataType.TEXT), + + // + TO_CHAR("to_char", YSQLDataType.TEXT, YSQLDataType.BYTEA, YSQLDataType.TEXT) { + @Override + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = super.getArguments(returnType, gen, depth); + args[0] = gen.generateExpression(YSQLDataType.getRandomType()); + return args; + } + }, + + // String functions + ASCII("ascii", YSQLDataType.INT, YSQLDataType.TEXT), + BTRIM("btrim", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + CHR("chr", YSQLDataType.TEXT, YSQLDataType.INT), + CONVERT_FROM("convert_from", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT) { + @Override + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = super.getArguments(returnType, gen, depth); + args[1] = YSQLConstant.createTextConstant("UTF8"); + return args; + } + }, + // concat + // segfault + BIT_LENGTH("bit_length", YSQLDataType.INT, YSQLDataType.BYTEA), + INITCAP("initcap", YSQLDataType.TEXT, YSQLDataType.TEXT), + LEFT("left", YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.TEXT), + LOWER("lower", YSQLDataType.TEXT, YSQLDataType.TEXT), MD5("md5", YSQLDataType.TEXT, YSQLDataType.TEXT), + UPPER("upper", YSQLDataType.TEXT, YSQLDataType.TEXT), + // PG_CLIENT_ENCODING("pg_client_encoding", YSQLDataType.TEXT), + QUOTE_LITERAL("quote_literal", YSQLDataType.TEXT, YSQLDataType.TEXT), + QUOTE_IDENT("quote_ident", YSQLDataType.TEXT, YSQLDataType.TEXT), + REGEX_REPLACE("regexp_replace", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + // todo mute repeat function because it may provide OOMs + // REPEAT("repeat", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + REPLACE("replace", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + REVERSE("reverse", YSQLDataType.TEXT, YSQLDataType.TEXT), + RIGHT("right", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + RPAD("rpad", YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.TEXT), + RTRIM("rtrim", YSQLDataType.TEXT, YSQLDataType.TEXT), + SPLIT_PART("split_part", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + STRPOS("strpos", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.TEXT), + SUBSTR("substr", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.INT), + TO_ASCII("to_ascii", YSQLDataType.TEXT, YSQLDataType.TEXT), TO_HEX("to_hex", YSQLDataType.INT, YSQLDataType.TEXT), + TRANSLATE("translate", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + // mathematical functions + // https://www.postgresql.org/docs/9.5/functions-math.html + ABS("abs", YSQLDataType.REAL, YSQLDataType.REAL), CBRT("cbrt", YSQLDataType.REAL, YSQLDataType.REAL), + CEILING("ceiling", YSQLDataType.REAL), // + DEGREES("degrees", YSQLDataType.REAL), EXP("exp", YSQLDataType.REAL), LN("ln", YSQLDataType.REAL), + LOG("log", YSQLDataType.REAL), LOG2("log", YSQLDataType.REAL, YSQLDataType.REAL), PI("pi", YSQLDataType.REAL), + POWER("power", YSQLDataType.REAL, YSQLDataType.REAL), TRUNC("trunc", YSQLDataType.REAL, YSQLDataType.INT), + TRUNC2("trunc", YSQLDataType.REAL, YSQLDataType.INT, YSQLDataType.REAL), FLOOR("floor", YSQLDataType.REAL), + + // trigonometric functions - complete + // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE + ACOS("acos", YSQLDataType.REAL), // + ACOSD("acosd", YSQLDataType.REAL), // + ASIN("asin", YSQLDataType.REAL), // + ASIND("asind", YSQLDataType.REAL), // + ATAN("atan", YSQLDataType.REAL), // + ATAND("atand", YSQLDataType.REAL), // + ATAN2("atan2", YSQLDataType.REAL, YSQLDataType.REAL), // + ATAN2D("atan2d", YSQLDataType.REAL, YSQLDataType.REAL), // + COS("cos", YSQLDataType.REAL), // + COSD("cosd", YSQLDataType.REAL), // + COT("cot", YSQLDataType.REAL), // + COTD("cotd", YSQLDataType.REAL), // + SIN("sin", YSQLDataType.REAL), // + SIND("sind", YSQLDataType.REAL), // + TAN("tan", YSQLDataType.REAL), // + TAND("tand", YSQLDataType.REAL), // + + // hyperbolic functions - complete + // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-HYP-TABLE + SINH("sinh", YSQLDataType.REAL), // + COSH("cosh", YSQLDataType.REAL), // + TANH("tanh", YSQLDataType.REAL), // + ASINH("asinh", YSQLDataType.REAL), // + ACOSH("acosh", YSQLDataType.REAL), // + ATANH("atanh", YSQLDataType.REAL), // + + // https://www.postgresql.org/docs/devel/functions-binarystring.html + GET_BIT("get_bit", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.INT), + GET_BYTE("get_byte", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.INT), + + // range functions + // https://www.postgresql.org/docs/devel/functions-range.html#RANGE-FUNCTIONS-TABLE + RANGE_LOWER("lower", YSQLDataType.INT, YSQLDataType.RANGE), // + RANGE_UPPER("upper", YSQLDataType.INT, YSQLDataType.RANGE), // + RANGE_ISEMPTY("isempty", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_LOWER_INC("lower_inc", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_UPPER_INC("upper_inc", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_LOWER_INF("lower_inf", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_UPPER_INF("upper_inf", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_MERGE("range_merge", YSQLDataType.RANGE, YSQLDataType.RANGE, YSQLDataType.RANGE), // + + // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE + GET_COLUMN_SIZE("get_column_size", YSQLDataType.INT, YSQLDataType.TEXT); + // PG_DATABASE_SIZE("pg_database_size", YSQLDataType.INT, YSQLDataType.INT); + // PG_SIZE_BYTES("pg_size_bytes", YSQLDataType.INT, YSQLDataType.TEXT); + + private final String functionName; + private final YSQLDataType returnType; + private final YSQLDataType[] argTypes; + + YSQLFunctionWithUnknownResult(String functionName, YSQLDataType returnType, YSQLDataType... indexType) { + this.functionName = functionName; + this.returnType = returnType; + this.argTypes = indexType.clone(); + } + + public static List getSupportedFunctions(YSQLDataType type) { + List functions = new ArrayList<>(); + for (YSQLFunctionWithUnknownResult func : values()) { + if (func.isCompatibleWithReturnType(type)) { + functions.add(func); + } + } + return functions; + } + + public boolean isCompatibleWithReturnType(YSQLDataType t) { + return t == returnType; + } + + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = new YSQLExpression[argTypes.length]; + for (int i = 0; i < args.length; i++) { + args[i] = gen.generateExpression(depth, argTypes[i]); + } + return args; + + } + + public String getName() { + return functionName; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java new file mode 100644 index 000000000..a8c5b9490 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java @@ -0,0 +1,65 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLInOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final List listElements; + private final boolean isTrue; + + public YSQLInOperation(YSQLExpression expr, List listElements, boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public YSQLExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + public boolean isTrue() { + return isTrue; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftValue = expr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return YSQLConstant.createNullConstant(); + } + boolean isNull = false; + for (YSQLExpression expr : getListElements()) { + YSQLConstant rightExpectedValue = expr.getExpectedValue(); + if (rightExpectedValue == null) { + return null; + } + if (rightExpectedValue.isNull()) { + isNull = true; + } else if (rightExpectedValue.isEquals(this.expr.getExpectedValue()).isBoolean() + && rightExpectedValue.isEquals(this.expr.getExpectedValue()).asBoolean()) { + return YSQLConstant.createBooleanConstant(isTrue); + } + } + + if (isNull) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(!isTrue); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java new file mode 100644 index 000000000..4edcbd11a --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java @@ -0,0 +1,56 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public class YSQLJoin implements YSQLExpression, Join { + + private final YSQLExpression tableReference; + private YSQLExpression onClause; + private final YSQLJoinType type; + + public YSQLJoin(YSQLExpression tableReference, YSQLExpression onClause, YSQLJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; + } + + public YSQLExpression getTableReference() { + return tableReference; + } + + public YSQLExpression getOnClause() { + return onClause; + } + + public YSQLJoinType getType() { + return type; + } + + @Override + public YSQLDataType getExpressionType() { + throw new AssertionError(); + } + + @Override + public YSQLConstant getExpectedValue() { + throw new AssertionError(); + } + + public enum YSQLJoinType { + INNER, LEFT, RIGHT, FULL, CROSS; + + public static YSQLJoinType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + @Override + public void setOnClause(YSQLExpression onClause) { + this.onClause = onClause; + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java b/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java new file mode 100644 index 000000000..e57a347f9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java @@ -0,0 +1,42 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLOrderByTerm implements YSQLExpression { + + private final YSQLOrder order; + private final YSQLExpression expr; + + public YSQLOrderByTerm(YSQLExpression expr, YSQLOrder order) { + this.expr = expr; + this.order = order; + } + + public YSQLOrder getOrder() { + return order; + } + + public YSQLExpression getExpr() { + return expr; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + @Override + public YSQLConstant getExpectedValue() { + throw new AssertionError(this); + } + + public enum YSQLOrder { + ASC, DESC; + + public static YSQLOrder getRandomOrder() { + return Randomly.fromOptions(YSQLOrder.values()); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java new file mode 100644 index 000000000..975ac6aaf --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java @@ -0,0 +1,65 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPOSIXRegularExpression implements YSQLExpression { + + private final YSQLExpression string; + private final YSQLExpression regex; + private final POSIXRegex op; + + public YSQLPOSIXRegularExpression(YSQLExpression string, YSQLExpression regex, POSIXRegex op) { + this.string = string; + this.regex = regex; + this.op = op; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + + public YSQLExpression getRegex() { + return regex; + } + + public YSQLExpression getString() { + return string; + } + + public POSIXRegex getOp() { + return op; + } + + public enum POSIXRegex implements Operator { + MATCH_CASE_SENSITIVE("~"), MATCH_CASE_INSENSITIVE("~*"), NOT_MATCH_CASE_SENSITIVE("!~"), + NOT_MATCH_CASE_INSENSITIVE("!~*"); + + private final String repr; + + POSIXRegex(String repr) { + this.repr = repr; + } + + public static POSIXRegex getRandom() { + return Randomly.fromOptions(values()); + } + + public String getStringRepresentation() { + return repr; + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java new file mode 100644 index 000000000..65ab056c5 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java @@ -0,0 +1,146 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPostfixOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final PostfixOperator op; + private final String operatorTextRepresentation; + + public YSQLPostfixOperation(YSQLExpression expr, PostfixOperator op) { + this.expr = expr; + this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); + this.op = op; + } + + public static YSQLExpression create(YSQLExpression expr, PostfixOperator op) { + return new YSQLPostfixOperation(expr, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.apply(expectedValue); + } + + public String getOperatorTextRepresentation() { + return operatorTextRepresentation; + } + + public YSQLExpression getExpression() { + return expr; + } + + public enum PostfixOperator implements Operator { + IS_NULL("IS NULL", "ISNULL") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return YSQLDataType.values(); + } + + }, + IS_UNKNOWN("IS UNKNOWN") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + }, + + IS_NOT_NULL("IS NOT NULL", "NOTNULL") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return YSQLDataType.values(); + } + + }, + IS_NOT_UNKNOWN("IS NOT UNKNOWN") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + }, + IS_TRUE("IS TRUE") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createFalse(); + } else { + return YSQLConstant.createBooleanConstant(expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + + }, + IS_FALSE("IS FALSE") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createFalse(); + } else { + return YSQLConstant.createBooleanConstant(!expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + + }; + + private final String[] textRepresentations; + + PostfixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public static PostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract YSQLConstant apply(YSQLConstant expectedValue); + + public abstract YSQLDataType[] getInputDataTypes(); + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java new file mode 100644 index 000000000..af9e64498 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java @@ -0,0 +1,36 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPostfixText implements YSQLExpression { + + private final YSQLExpression expr; + private final String text; + private final YSQLConstant expectedValue; + private final YSQLDataType type; + + public YSQLPostfixText(YSQLExpression expr, String text, YSQLConstant expectedValue, YSQLDataType type) { + this.expr = expr; + this.text = text; + this.expectedValue = expectedValue; + this.type = type; + } + + public YSQLExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + @Override + public YSQLDataType getExpressionType() { + return type; + } + + @Override + public YSQLConstant getExpectedValue() { + return expectedValue; + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java new file mode 100644 index 000000000..d34f07567 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java @@ -0,0 +1,115 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPrefixOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final PrefixOperator op; + + public YSQLPrefixOperation(YSQLExpression expr, PrefixOperator op) { + this.expr = expr; + this.op = op; + } + + @Override + public YSQLDataType getExpressionType() { + return op.getExpressionType(); + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.getExpectedValue(expectedValue); + } + + public YSQLDataType[] getInputDataTypes() { + return op.dataTypes; + } + + public String getTextRepresentation() { + return op.textRepresentation; + } + + public YSQLExpression getExpression() { + return expr; + } + + public enum PrefixOperator implements Operator { + NOT("NOT", YSQLDataType.BOOLEAN) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(!expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + }, + UNARY_PLUS("+", YSQLDataType.INT) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + // TODO: actual converts to double precision + return expectedValue; + } + + }, + UNARY_MINUS("-", YSQLDataType.INT) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + // TODO + throw new IgnoreMeException(); + } + if (expectedValue.isInt() && expectedValue.asInt() == Long.MIN_VALUE) { + throw new IgnoreMeException(); + } + try { + return YSQLConstant.createIntConstant(-expectedValue.asInt()); + } catch (UnsupportedOperationException e) { + return null; + } + } + + }; + + private final String textRepresentation; + private final YSQLDataType[] dataTypes; + + PrefixOperator(String textRepresentation, YSQLDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract YSQLDataType getExpressionType(); + + protected abstract YSQLConstant getExpectedValue(YSQLConstant expectedValue); + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java new file mode 100644 index 000000000..fbf96ee45 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java @@ -0,0 +1,145 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; + +public class YSQLSelect extends SelectBase + implements YSQLExpression, Select { + + private SelectType selectOption = SelectType.ALL; + private List joinClauses = Collections.emptyList(); + private YSQLExpression distinctOnClause; + private ForClause forClause; + + public void setSelectType(SelectType fromOptions) { + this.setSelectOption(fromOptions); + } + + public SelectType getSelectOption() { + return selectOption; + } + + public void setSelectOption(SelectType fromOptions) { + this.selectOption = fromOptions; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + @Override + public List getJoinClauses() { + return joinClauses; + } + + @Override + public void setJoinClauses(List joinStatements) { + this.joinClauses = joinStatements; + + } + + public YSQLExpression getDistinctOnClause() { + return distinctOnClause; + } + + public void setDistinctOnClause(YSQLExpression distinctOnClause) { + if (selectOption != SelectType.DISTINCT) { + throw new IllegalArgumentException(); + } + this.distinctOnClause = distinctOnClause; + } + + public ForClause getForClause() { + return forClause; + } + + public void setForClause(ForClause forClause) { + this.forClause = forClause; + } + + public enum ForClause { + UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); + + private final String textRepresentation; + + ForClause(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ForClause getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + public enum SelectType { + DISTINCT, ALL; + + public static SelectType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class YSQLFromTable implements YSQLExpression { + private final YSQLTable t; + private final boolean only; + + public YSQLFromTable(YSQLTable t, boolean only) { + this.t = t; + this.only = only; + } + + public YSQLTable getTable() { + return t; + } + + public boolean isOnly() { + return only; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + } + + public static class YSQLSubquery implements YSQLExpression { + private final YSQLSelect s; + private final String name; + + public YSQLSubquery(YSQLSelect s, String name) { + this.s = s; + this.name = name; + } + + public YSQLSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + } + + @Override + public String asString() { + return YSQLVisitor.asString(this); + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java b/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java new file mode 100644 index 000000000..794525bc2 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java @@ -0,0 +1,39 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLSimilarTo implements YSQLExpression { + + private final YSQLExpression string; + private final YSQLExpression similarTo; + private final YSQLExpression escapeCharacter; + + public YSQLSimilarTo(YSQLExpression string, YSQLExpression similarTo, YSQLExpression escapeCharacter) { + this.string = string; + this.similarTo = similarTo; + this.escapeCharacter = escapeCharacter; + } + + public YSQLExpression getString() { + return string; + } + + public YSQLExpression getSimilarTo() { + return similarTo; + } + + public YSQLExpression getEscapeCharacter() { + return escapeCharacter; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java new file mode 100644 index 000000000..c4a4effd8 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java @@ -0,0 +1,183 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public class YSQLAlterTableGenerator { + + private final YSQLTable randomTable; + private final Randomly r; + private final YSQLGlobalState globalState; + + public YSQLAlterTableGenerator(YSQLTable randomTable, YSQLGlobalState globalState) { + this.randomTable = randomTable; + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter create(YSQLTable randomTable, YSQLGlobalState globalState) { + return new YSQLAlterTableGenerator(randomTable, globalState).generate(); + } + + public List getActions(ExpectedErrors errors) { + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonInsertUpdateErrors(errors); + YSQLErrors.addCommonTableErrors(errors); + errors.add("duplicate key value violates unique constraint"); + errors.add("cannot drop key column"); + errors.add("cannot drop desired object(s) because other objects depend on them"); + errors.add("invalid input syntax for"); + errors.add("cannot remove a key column"); + errors.add("it has pending trigger events"); + errors.add("could not open relation"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not accept data type"); + errors.add("does not exist for access method"); + errors.add("could not find cast from"); + errors.add("does not exist"); // TODO: investigate + errors.add("constraints on permanent tables may reference only permanent tables"); + List action; + if (Randomly.getBoolean()) { + action = Randomly.nonEmptySubset(Action.values()); + } else { + // make it more likely that the ALTER TABLE succeeds + action = Randomly.subset(Randomly.smallNumber(), Action.values()); + } + if (randomTable.getColumns().size() == 1) { + action.remove(Action.ALTER_TABLE_DROP_COLUMN); + } + if (!randomTable.hasIndexes()) { + action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); + } + if (action.isEmpty()) { + throw new IgnoreMeException(); + } + return action; + } + + public SQLQueryAdapter generate() { + ExpectedErrors errors = new ExpectedErrors(); + int i = 0; + List action = getActions(errors); + StringBuilder sb = new StringBuilder(); + sb.append("ALTER TABLE "); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + errors.add("cannot use ONLY for foreign key on partitioned table"); + } + sb.append(" "); + sb.append(randomTable.getName()); + sb.append(" "); + for (Action a : action) { + if (i++ != 0) { + sb.append(", "); + } + switch (a) { + case ALTER_TABLE_DROP_COLUMN: + sb.append("DROP "); + if (Randomly.getBoolean()) { + sb.append(" IF EXISTS "); + } + sb.append(randomTable.getRandomColumn().getName()); + errors.add("because other objects depend on it"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("RESTRICT", "CASCADE")); + } + errors.add("does not exist"); + errors.add("cannot drop column"); + errors.add("cannot drop key column"); + errors.add("cannot drop inherited column"); + break; + case ADD_TABLE_CONSTRAINT: + sb.append("ADD "); + sb.append("CONSTRAINT ").append(r.getAlphabeticChar()).append(" "); + YSQLCommon.addTableConstraint(sb, randomTable, globalState, errors); + errors.add("already exists"); + errors.add("multiple primary keys for table"); + errors.add("could not create unique index"); + errors.add("contains null values"); + errors.add("cannot cast type"); + errors.add("unsupported PRIMARY KEY constraint with partition key definition"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("which is part of the partition key"); + errors.add("out of range"); + errors.add("there is no unique constraint matching given keys for referenced table"); + errors.add("constraints on temporary tables may reference only temporary tables"); + errors.add("constraints on unlogged tables may reference only permanent or unlogged tables"); + errors.add("constraints on permanent tables may reference only permanent tables"); + errors.add("cannot reference partitioned table"); + errors.add("cannot be implemented"); + errors.add("violates foreign key constraint"); + errors.add("unsupported ON COMMIT and foreign key combination"); + errors.add("USING INDEX is not supported on partitioned tables"); + if (Randomly.getBoolean()) { + sb.append(" NOT VALID"); + errors.add("cannot be marked NOT VALID"); + errors.add("cannot add NOT VALID foreign key on partitioned table"); + } else { + errors.add("is violated by some row"); + } + break; + case ADD_TABLE_CONSTRAINT_USING_INDEX: + sb.append("ADD "); + sb.append("CONSTRAINT ").append(r.getAlphabeticChar()).append(" "); + sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); + sb.append(" USING INDEX "); + sb.append(randomTable.getRandomIndex().getIndexName()); + errors.add("already exists"); + errors.add("PRIMARY KEY containing column of type"); + errors.add("not valid"); + errors.add("is not a unique index"); + errors.add("is already associated with a constraint"); + errors.add("Cannot create a primary key or unique constraint using such an index"); + errors.add("multiple primary keys for table"); + errors.add("appears twice in unique constraint"); + errors.add("appears twice in primary key constraint"); + errors.add("contains null values"); + errors.add("insufficient columns in PRIMARY KEY constraint definition"); + errors.add("which is part of the partition key"); + break; + case DISABLE_ROW_LEVEL_SECURITY: + sb.append("DISABLE ROW LEVEL SECURITY"); + break; + case ENABLE_ROW_LEVEL_SECURITY: + sb.append("ENABLE ROW LEVEL SECURITY"); + break; + case FORCE_ROW_LEVEL_SECURITY: + sb.append("FORCE ROW LEVEL SECURITY"); + break; + case NO_FORCE_ROW_LEVEL_SECURITY: + sb.append("NO FORCE ROW LEVEL SECURITY"); + break; + default: + throw new AssertionError(a); + } + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + protected enum Action { + // ALTER_TABLE_ADD_COLUMN, // [ COLUMN ] column data_type [ COLLATE collation ] [ + // column_constraint [ ... ] ] + ALTER_TABLE_DROP_COLUMN, // DROP [ COLUMN ] [ IF EXISTS ] column [ RESTRICT | CASCADE ] + ADD_TABLE_CONSTRAINT, // ADD table_constraint [ NOT VALID ] + ADD_TABLE_CONSTRAINT_USING_INDEX, // ADD table_constraint_using_index + DISABLE_ROW_LEVEL_SECURITY, // DISABLE ROW LEVEL SECURITY + ENABLE_ROW_LEVEL_SECURITY, // ENABLE ROW LEVEL SECURITY + FORCE_ROW_LEVEL_SECURITY, // FORCE ROW LEVEL SECURITY + NO_FORCE_ROW_LEVEL_SECURITY, // NO FORCE ROW LEVEL SECURITY + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java new file mode 100644 index 000000000..bee2ec5ef --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java @@ -0,0 +1,37 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLAnalyzeGenerator { + + private YSQLAnalyzeGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + YSQLTable table = globalState.getSchema().getRandomTable(); + StringBuilder sb = new StringBuilder("ANALYZE"); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(" VERBOSE"); + sb.append(")"); + } + sb.append(" "); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(table.getRandomNonEmptyColumnSubset().stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + } + + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("deadlock")); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java new file mode 100644 index 000000000..6e852d167 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java @@ -0,0 +1,32 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLClusterGenerator { + + private YSQLClusterGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("there is no previously clustered index for table"); + errors.add("cannot cluster a partitioned table"); + errors.add("access method does not support clustering"); + StringBuilder sb = new StringBuilder("CLUSTER "); + if (Randomly.getBoolean()) { + YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean() && !table.getIndexes().isEmpty()) { + sb.append(" USING "); + sb.append(table.getRandomIndex().getIndexName()); + errors.add("cannot cluster on partial index"); + } + } + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java new file mode 100644 index 000000000..f020a8994 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java @@ -0,0 +1,68 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +/** + * @see COMMENT + */ +public final class YSQLCommentGenerator { + + private YSQLCommentGenerator() { + } + + public static SQLQueryAdapter generate(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("COMMENT ON "); + Action type = Randomly.fromOptions(Action.values()); + YSQLTable randomTable = globalState.getSchema().getRandomTable(); + switch (type) { + case INDEX: + sb.append("INDEX "); + if (randomTable.getIndexes().isEmpty()) { + throw new IgnoreMeException(); + } else { + sb.append(randomTable.getRandomIndex().getIndexName()); + } + break; + case COLUMN: + sb.append("COLUMN "); + sb.append(randomTable.getRandomColumn().getFullQualifiedName()); + break; + case STATISTICS: + sb.append("STATISTICS "); + if (randomTable.getStatistics().isEmpty()) { + throw new IgnoreMeException(); + } else { + sb.append(randomTable.getStatistics().get(0).getName()); + } + break; + case TABLE: + sb.append("TABLE "); + if (randomTable.isView()) { + throw new IgnoreMeException(); + } + sb.append(randomTable.getName()); + break; + default: + throw new AssertionError(type); + } + sb.append(" IS "); + if (Randomly.getBoolean()) { + sb.append("NULL"); + } else { + sb.append("'"); + sb.append(globalState.getRandomly().getString().replace("'", "''")); + sb.append("'"); + } + return new SQLQueryAdapter(sb.toString()); + } + + private enum Action { + INDEX, COLUMN, STATISTICS, TABLE + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java new file mode 100644 index 000000000..231bf8008 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -0,0 +1,283 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; + +public final class YSQLCommon { + + private YSQLCommon() { + } + + public static boolean appendDataType(YSQLDataType type, StringBuilder sb, boolean allowSerial, + boolean generateOnlyKnown, List opClasses) throws AssertionError { + boolean serial = false; + switch (type) { + case BOOLEAN: + sb.append("boolean"); + break; + case INT: + if (Randomly.getBoolean() && allowSerial) { + serial = true; + sb.append(Randomly.fromOptions("serial", "bigserial")); + } else { + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + } + break; + case TEXT: + if (Randomly.getBoolean()) { + sb.append("TEXT"); + } else if (Randomly.getBoolean()) { + // TODO: support CHAR (without VAR) + if (YSQLProvider.generateOnlyKnown || Randomly.getBoolean()) { + sb.append("VAR"); + } + sb.append("CHAR"); + sb.append("("); + sb.append(ThreadLocalRandom.current().nextInt(1, 500)); + sb.append(")"); + } else { + sb.append("name"); + } + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + case REAL: + if (Randomly.getBoolean()) { + sb.append("REAL"); + } else { + sb.append("FLOAT"); + } + break; + case RANGE: + sb.append(Randomly.fromOptions("int4range", "int4range")); // , "int8range", "numrange" + break; + case MONEY: + sb.append("money"); + break; + case BYTEA: + sb.append("bytea"); + break; + case BIT: + sb.append("BIT"); + // if (Randomly.getBoolean()) { + sb.append(" VARYING"); + // } + sb.append("("); + sb.append(Randomly.getNotCachedInteger(1, 500)); + sb.append(")"); + break; + case INET: + sb.append("inet"); + break; + default: + throw new AssertionError(type); + } + return serial; + } + + public static void generateWith(StringBuilder sb, YSQLGlobalState globalState, ExpectedErrors errors, + List columnsToBeAdded, boolean isTemporaryTable) { + if (Randomly.getBoolean()) { + sb.append(" WITHOUT OIDS "); + } else if (Randomly.getBoolean() && !isTemporaryTable) { + if (Randomly.getBoolean()) { + sb.append(" SPLIT INTO "); + sb.append(Randomly.smallNumber() + 1); + sb.append(" TABLETS "); + + errors.add("cannot create colocated table with split option"); + errors.add("columns must be present to split by number of tablets"); + errors.add("option is not yet supported for hash partitioned tables"); + } else { + sb.append(" SPLIT AT VALUES ("); + + errors.add("cannot create colocated table with split option"); + errors.add("SPLIT AT option is not yet supported for hash partitioned tables"); + errors.add("Cannot have duplicate split rows"); // just in case + + boolean hasBoolean = false; + for (YSQLColumn column : columnsToBeAdded) { + if (column.getType().equals(YSQLDataType.BOOLEAN)) { + hasBoolean = true; + break; + } + } + + int splits = hasBoolean ? 2 : Randomly.smallNumber() + 2; + long start = Randomly.smallNumber(); + boolean[] bools = { false, true }; + for (int i = 1; i <= splits; i++) { + int size = columnsToBeAdded.size(); + int counter = 1; + for (YSQLColumn c : columnsToBeAdded) { + sb.append("("); + switch (c.getType()) { + case INT: + case REAL: + sb.append(YSQLConstant.createDoubleConstant(start)); + case FLOAT: + sb.append(YSQLConstant.createIntConstant(start)); + break; + case BOOLEAN: + sb.append(YSQLConstant.createBooleanConstant(bools[i - 1])); + break; + case TEXT: + sb.append(YSQLConstant.createTextConstant(String.valueOf(start))); + break; + default: + throw new IgnoreMeException(); + } + sb.append(")"); + counter++; + start += Randomly.smallNumber() + 1; + if (counter <= size) { + sb.append(","); + } + } + + if (i < splits) { + sb.append(","); + } + } + sb.append(")"); + } + } else if (Randomly.getBoolean()) { + errors.add("Cannot use TABLEGROUP with TEMP table"); + if (!globalState.getSchema().getDatabaseIsColocated(globalState.getConnection())) { + sb.append(" TABLEGROUP tg").append( + Randomly.getNotCachedInteger(1, (int) YSQLTableGroupGenerator.UNIQUE_TABLEGROUP_COUNTER.get())); + } + } + } + + public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder sb, YSQLTable table, + YSQLGlobalState globalState, ExpectedErrors errors) { + // TODO constraint name + List tableConstraints = Randomly.nonEmptySubset(TableConstraints.values()); + if (excludePrimaryKey) { + tableConstraints.remove(TableConstraints.PRIMARY_KEY); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + tableConstraints.remove(TableConstraints.FOREIGN_KEY); + } + for (TableConstraints t : tableConstraints) { + sb.append(", "); + // TODO add index parameters + addTableConstraint(sb, table, globalState, t, errors); + } + } + + public static void addTableConstraint(StringBuilder sb, YSQLTable table, YSQLGlobalState globalState, + ExpectedErrors errors) { + addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); + } + + private static void addTableConstraint(StringBuilder sb, YSQLTable table, YSQLGlobalState globalState, + TableConstraints t, ExpectedErrors errors) { + List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); + List otherColumns; + YSQLErrors.addCommonExpressionErrors(errors); + switch (t) { + case CHECK: + sb.append("CHECK("); + sb.append(YSQLVisitor.getExpressionAsString(globalState, YSQLDataType.BOOLEAN, table.getColumns())); + sb.append(")"); + errors.add("constraint must be added to child tables too"); + errors.add("missing FROM-clause entry for table"); + break; + case UNIQUE: + sb.append("UNIQUE("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + break; + case FOREIGN_KEY: + sb.append("FOREIGN KEY ("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(") REFERENCES "); + YSQLTable randomOtherTable = globalState.getSchema().getRandomTable(tab -> !tab.isView()); + sb.append(randomOtherTable.getName()); + if (randomOtherTable.getColumns().size() < randomNonEmptyColumnSubset.size()) { + throw new IgnoreMeException(); + } + otherColumns = randomOtherTable.getRandomNonEmptyColumnSubset(randomNonEmptyColumnSubset.size()); + sb.append("("); + sb.append(otherColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("MATCH FULL", "MATCH SIMPLE")); + } + if (Randomly.getBoolean()) { + sb.append(" ON DELETE "); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" ON UPDATE "); + errors.add("invalid ON UPDATE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" "); + if (Randomly.getBoolean()) { + sb.append("DEFERRABLE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("INITIALLY DEFERRED", "INITIALLY IMMEDIATE")); + } + } else { + sb.append("NOT DEFERRABLE"); + } + } + break; + default: + throw new AssertionError(t); + } + } + + private static void deleteOrUpdateAction(StringBuilder sb) { + sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); + } + + public enum TableConstraints { + CHECK, UNIQUE, PRIMARY_KEY, FOREIGN_KEY + } + + // private enum StorageParameters { + // COLOCATED("COLOCATED", (r) -> Randomly.getBoolean()); + // // TODO + // + // private final String parameter; + // private final Function op; + // + // StorageParameters(String parameter, Function op) { + // this.parameter = parameter; + // this.op = op; + // } + // } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java new file mode 100644 index 000000000..9d3c7427d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -0,0 +1,46 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractDeleteGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; + +public final class YSQLDeleteGenerator extends AbstractDeleteGenerator { + + private final YSQLGlobalState globalState; + + private YSQLDeleteGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + return new YSQLDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + errors.add("violates foreign key constraint"); + errors.add("violates not-null constraint"); + errors.add("could not determine which collation to use for string comparison"); + appendDeleteFromTable(table.getName(), Randomly.getBoolean()); + if (Randomly.getBoolean()) { + appendWhereClause(YSQLVisitor.asString( + YSQLExpressionGenerator.generateExpression(globalState, table.getColumns(), YSQLDataType.BOOLEAN))); + } + if (Randomly.getBoolean()) { + appendReturningClause( + YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, table.getColumns()))); + } + YSQLErrors.addCommonExpressionErrors(errors); + errors.add("out of range"); + errors.add("cannot cast"); + errors.add("invalid input syntax for"); + errors.add("division by zero"); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java new file mode 100644 index 000000000..156fb97da --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java @@ -0,0 +1,40 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable.TableType; + +public final class YSQLDiscardGenerator { + + private YSQLDiscardGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("DISCARD "); + // prevent that DISCARD discards all tables (if they are TEMP tables) + boolean hasNonTempTables = globalState.getSchema().getDatabaseTables().stream() + .anyMatch(t -> t.getTableType() == TableType.STANDARD); + String what; + if (hasNonTempTables) { + what = Randomly.fromOptions("ALL", "PLANS", "SEQUENCES", "TEMPORARY", "TEMP"); + } else { + what = Randomly.fromOptions("PLANS", "SEQUENCES"); + } + sb.append(what); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + private static final long serialVersionUID = 1L; + + @Override + public boolean couldAffectSchema() { + return canDiscardTemporaryTables(what); + } + }; + } + + private static boolean canDiscardTemporaryTables(String what) { + return what.contentEquals("TEMPORARY") || what.contentEquals("TEMP") || what.contentEquals("ALL"); + } +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java new file mode 100644 index 000000000..da9b5ae61 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; + +public final class YSQLDropIndexGenerator { + + private YSQLDropIndexGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + List indexes = globalState.getSchema().getRandomTable().getIndexes(); + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + if (Randomly.getBoolean() || indexes.isEmpty()) { + sb.append("IF EXISTS "); + if (indexes.isEmpty() || Randomly.getBoolean()) { + sb.append(DBMSCommon.createIndexName(Randomly.smallNumber())); + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", + "cannot drop index", "does not exist"), + true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java new file mode 100644 index 000000000..80c35c431 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java @@ -0,0 +1,689 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLRowValue; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryArithmeticOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryBitOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryComparisonOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryRangeOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConcatOperation; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLFunctionWithUnknownResult; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public class YSQLExpressionGenerator implements ExpressionGenerator, + NoRECGenerator, + TLPWhereGenerator { + + private final int maxDepth; + + private final Randomly r; + private final Map functionsAndTypes; + private final List allowedFunctionTypes; + private List columns; + private List tables; + private YSQLRowValue rw; + private boolean expectedResult; + private YSQLGlobalState globalState; + private boolean allowAggregateFunctions; + + public YSQLExpressionGenerator(YSQLGlobalState globalState) { + this.r = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + this.globalState = globalState; + this.functionsAndTypes = globalState.getFunctionsAndTypes(); + this.allowedFunctionTypes = globalState.getAllowedFunctionTypes(); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, YSQLDataType type) { + return new YSQLExpressionGenerator(globalState).generateExpression(0, type); + } + + private static YSQLCompoundDataType getCompoundDataType(YSQLDataType type) { + switch (type) { + case BOOLEAN: + case DECIMAL: // TODO + case FLOAT: + case INT: + case MONEY: + case RANGE: + case REAL: + case INET: + case BYTEA: + return YSQLCompoundDataType.create(type); + case TEXT: // TODO + case BIT: + if (Randomly.getBoolean() + || YSQLProvider.generateOnlyKnown /* + * The PQS implementation does not check for size specifications + */) { + return YSQLCompoundDataType.create(type); + } else { + return YSQLCompoundDataType.create(type, (int) Randomly.getNotCachedInteger(1, 1000)); + } + default: + throw new AssertionError(type); + } + + } + + public static YSQLExpression generateConstant(Randomly r, YSQLDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return YSQLConstant.createNullConstant(); + } + // if (Randomly.getBooleanWithSmallProbability()) { + // return YSQLConstant.createTextConstant(r.getString()); + // } + switch (type) { + case INT: + if (Randomly.getBooleanWithSmallProbability()) { + return YSQLConstant.createTextConstant(String.valueOf(r.getInteger())); + } else { + return YSQLConstant.createIntConstant(r.getInteger()); + } + case BOOLEAN: + if (Randomly.getBooleanWithSmallProbability() && !YSQLProvider.generateOnlyKnown) { + return YSQLConstant + .createTextConstant(Randomly.fromOptions("TR", "TRUE", "FA", "FALSE", "0", "1", "ON", "off")); + } else { + return YSQLConstant.createBooleanConstant(Randomly.getBoolean()); + } + case TEXT: + return YSQLConstant.createTextConstant(r.getString()); + case DECIMAL: + return YSQLConstant.createDecimalConstant(r.getRandomBigDecimal()); + case FLOAT: + return YSQLConstant.createFloatConstant((float) r.getDouble()); + case REAL: + return YSQLConstant.createDoubleConstant(r.getDouble()); + case RANGE: + return YSQLConstant.createRange(r.getInteger(), Randomly.getBoolean(), r.getInteger(), + Randomly.getBoolean()); + case MONEY: + return new YSQLCastOperation(generateConstant(r, YSQLDataType.FLOAT), + getCompoundDataType(YSQLDataType.MONEY)); + case INET: + return YSQLConstant.createInetConstant(getRandomInet(r)); + case BIT: + return YSQLConstant.createBitConstant(r.getInteger()); + case BYTEA: + return YSQLConstant.createByteConstant(String.valueOf(r.getInteger())); + default: + throw new AssertionError(type); + } + } + + private static String getRandomInet(Randomly r) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 4; i++) { + if (i != 0) { + sb.append('.'); + } + sb.append(r.getInteger() & 255); + } + return sb.toString(); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, List columns, + YSQLDataType type) { + return new YSQLExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, List columns) { + return new YSQLExpressionGenerator(globalState).setColumns(columns).generateExpression(0); + + } + + public YSQLExpressionGenerator setColumns(List columns) { + this.columns = columns; + return this; + } + + public YSQLExpressionGenerator setRowValue(YSQLRowValue rw) { + this.rw = rw; + return this; + } + + public YSQLExpression generateExpression(int depth) { + return generateExpression(depth, YSQLDataType.getRandomType()); + } + + @Override + public List generateOrderBys() { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new YSQLOrderByTerm(YSQLColumnValue.create(Randomly.fromList(columns), null), + YSQLOrderByTerm.YSQLOrder.getRandomOrder())); + } + return orderBys; + } + + private YSQLExpression generateFunctionWithUnknownResult(int depth, YSQLDataType type) { + List supportedFunctions = YSQLFunctionWithUnknownResult + .getSupportedFunctions(type); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + supportedFunctions = supportedFunctions.stream() + .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (supportedFunctions.isEmpty()) { + throw new IgnoreMeException(); + } + YSQLFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); + return new YSQLFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); + } + + private YSQLExpression generateFunctionWithKnownResult(int depth, YSQLDataType type) { + List functions = Stream.of(YSQLFunction.YSQLFunctionWithResult.values()) + .filter(f -> f.supportsReturnType(type)).collect(Collectors.toList()); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (functions.isEmpty()) { + throw new IgnoreMeException(); + } + YSQLFunction.YSQLFunctionWithResult randomFunction = Randomly.fromList(functions); + int nrArgs = randomFunction.getNrArgs(); + if (randomFunction.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + YSQLDataType[] argTypes = randomFunction.getInputTypesForReturnType(type, nrArgs); + YSQLExpression[] args = new YSQLExpression[nrArgs]; + do { + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1, argTypes[i]); + } + } while (!randomFunction.checkArguments(args)); + return new YSQLFunction(randomFunction, type, args); + } + + private YSQLExpression generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (YSQLProvider.generateOnlyKnown) { + validOptions.remove(BooleanExpression.SIMILAR_TO); + validOptions.remove(BooleanExpression.POSIX_REGEX); + validOptions.remove(BooleanExpression.BINARY_RANGE_COMPARISON); + } + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + YSQLPostfixOperation.PostfixOperator random = YSQLPostfixOperation.PostfixOperator.getRandom(); + return YSQLPostfixOperation + .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); + case IN_OPERATION: + return inOperation(depth + 1); + case NOT: + return new YSQLPrefixOperation(generateExpression(depth + 1, YSQLDataType.BOOLEAN), + YSQLPrefixOperation.PrefixOperator.NOT); + case BINARY_LOGICAL_OPERATOR: + YSQLExpression first = generateExpression(depth + 1, YSQLDataType.BOOLEAN); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + first = new YSQLBinaryLogicalOperation(first, generateExpression(depth + 1, YSQLDataType.BOOLEAN), + YSQLBinaryLogicalOperation.BinaryLogicalOperator.getRandom()); + } + return first; + case BINARY_COMPARISON: + YSQLDataType dataType = getMeaningfulType(); + return generateComparison(depth, dataType); + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.BOOLEAN)); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.BOOLEAN); + case BETWEEN: + YSQLDataType type = getMeaningfulType(); + return new YSQLBetweenOperation(generateExpression(depth + 1, type), generateExpression(depth + 1, type), + generateExpression(depth + 1, type), Randomly.getBoolean()); + case SIMILAR_TO: + assert !expectedResult; + // TODO also generate the escape character + return new YSQLSimilarTo(generateExpression(depth + 1, YSQLDataType.TEXT), + generateExpression(depth + 1, YSQLDataType.TEXT), null); + case POSIX_REGEX: + assert !expectedResult; + return new YSQLPOSIXRegularExpression(generateExpression(depth + 1, YSQLDataType.TEXT), + generateExpression(depth + 1, YSQLDataType.TEXT), + YSQLPOSIXRegularExpression.POSIXRegex.getRandom()); + case BINARY_RANGE_COMPARISON: + // TODO element check + return new YSQLBinaryRangeOperation(YSQLBinaryRangeOperation.YSQLBinaryRangeComparisonOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.RANGE), + generateExpression(depth + 1, YSQLDataType.RANGE)); + default: + throw new AssertionError(); + } + } + + private YSQLDataType getMeaningfulType() { + // make it more likely that the expression does not only consist of constant + // expressions + if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { + return YSQLDataType.getRandomType(); + } else { + return Randomly.fromList(columns).getType(); + } + } + + private YSQLExpression generateFunction(int depth, YSQLDataType type) { + if (YSQLProvider.generateOnlyKnown || Randomly.getBoolean()) { + return generateFunctionWithKnownResult(depth, type); + } else { + return generateFunctionWithUnknownResult(depth, type); + } + } + + private YSQLExpression generateComparison(int depth, YSQLDataType dataType) { + YSQLExpression leftExpr = generateExpression(depth + 1, dataType); + YSQLExpression rightExpr = generateExpression(depth + 1, dataType); + return getComparison(leftExpr, rightExpr); + } + + private YSQLExpression getComparison(YSQLExpression leftExpr, YSQLExpression rightExpr) { + return new YSQLBinaryComparisonOperation(leftExpr, rightExpr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.getRandom()); + } + + private YSQLExpression inOperation(int depth) { + YSQLDataType type = YSQLDataType.getRandomType(); + YSQLExpression leftExpr = generateExpression(depth + 1, type); + List rightExpr = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + rightExpr.add(generateExpression(depth + 1, type)); + } + return new YSQLInOperation(leftExpr, rightExpr, Randomly.getBoolean()); + } + + public YSQLExpression generateExpression(int depth, YSQLDataType originalType) { + YSQLDataType dataType = originalType; + if (dataType == YSQLDataType.REAL && Randomly.getBoolean()) { + dataType = Randomly.fromOptions(YSQLDataType.INT, YSQLDataType.FLOAT); + } + if (dataType == YSQLDataType.FLOAT && Randomly.getBoolean()) { + dataType = YSQLDataType.INT; + } + return generateExpressionInternal(depth, dataType); + } + + private YSQLExpression generateExpressionInternal(int depth, YSQLDataType dataType) throws AssertionError { + if (allowAggregateFunctions && Randomly.getBoolean()) { + allowAggregateFunctions = false; // aggregate function calls cannot be nested + return getAggregate(dataType); + } + if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { + // generic expression + if (Randomly.getBoolean() || depth > maxDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(r, dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(r, dataType); + } else { + return createColumnOfType(dataType); + } + } + } else { + if (Randomly.getBoolean()) { + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(dataType)); + } else { + return generateFunctionWithUnknownResult(depth, dataType); + } + } + } else { + switch (dataType) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case TEXT: + return generateTextExpression(depth); + case DECIMAL: + case REAL: + case FLOAT: + case MONEY: + case INET: + return generateConstant(r, dataType); + case BYTEA: + return generateByteExpression(); + case BIT: + return generateBitExpression(depth); + case RANGE: + return generateRangeExpression(depth); + default: + throw new AssertionError(dataType); + } + } + } + + private YSQLExpression generateRangeExpression(int depth) { + RangeExpression option; + List validOptions = new ArrayList<>(Arrays.asList(RangeExpression.values())); + option = Randomly.fromList(validOptions); + switch (option) { + case BINARY_OP: + return new YSQLBinaryRangeOperation(YSQLBinaryRangeOperation.YSQLBinaryRangeOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.RANGE), + generateExpression(depth + 1, YSQLDataType.RANGE)); + default: + throw new AssertionError(option); + } + } + + private YSQLExpression generateTextExpression(int depth) { + TextExpression option; + List validOptions = new ArrayList<>(Arrays.asList(TextExpression.values())); + option = Randomly.fromList(validOptions); + + switch (option) { + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.TEXT)); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.TEXT); + case CONCAT: + return generateConcat(depth); + default: + throw new AssertionError(); + } + } + + private YSQLExpression generateConcat(int depth) { + YSQLExpression left = generateExpression(depth + 1, YSQLDataType.TEXT); + YSQLExpression right = generateExpression(depth + 1); + return new YSQLConcatOperation(left, right); + } + + private YSQLExpression generateByteExpression() { + return YSQLConstant.createByteConstant("Th\\000omas"); + } + + private YSQLExpression generateBitExpression(int depth) { + BitExpression option; + option = Randomly.fromOptions(BitExpression.values()); + switch (option) { + case BINARY_OPERATION: + return new YSQLBinaryBitOperation(YSQLBinaryBitOperation.YSQLBinaryBitOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.BIT), generateExpression(depth + 1, YSQLDataType.BIT)); + default: + throw new AssertionError(); + } + } + + private YSQLExpression generateIntExpression(int depth) { + IntExpression option; + option = Randomly.fromOptions(IntExpression.values()); + switch (option) { + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.INT)); + case UNARY_OPERATION: + YSQLExpression intExpression = generateExpression(depth + 1, YSQLDataType.INT); + return new YSQLPrefixOperation(intExpression, Randomly.getBoolean() + ? YSQLPrefixOperation.PrefixOperator.UNARY_PLUS : YSQLPrefixOperation.PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.INT); + case BINARY_ARITHMETIC_EXPRESSION: + return new YSQLBinaryArithmeticOperation(generateExpression(depth + 1, YSQLDataType.INT), + generateExpression(depth + 1, YSQLDataType.INT), + YSQLBinaryArithmeticOperation.YSQLBinaryOperator.getRandom()); + default: + throw new AssertionError(); + } + } + + private YSQLExpression createColumnOfType(YSQLDataType type) { + List columns = filterColumns(type); + YSQLColumn fromList = Randomly.fromList(columns); + YSQLConstant value = rw == null ? null : rw.getValues().get(fromList); + return YSQLColumnValue.create(fromList, value); + } + + final List filterColumns(YSQLDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + public YSQLExpression generateExpressionWithExpectedResult(YSQLDataType type) { + this.expectedResult = true; + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns).setRowValue(rw); + YSQLExpression expr; + do { + expr = gen.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + public List generateExpressions(int nr) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(0)); + } + return expressions; + } + + public YSQLExpression generateExpression(YSQLDataType dataType) { + return generateExpression(0, dataType); + } + + public YSQLExpressionGenerator setGlobalState(YSQLGlobalState globalState) { + this.globalState = globalState; + return this; + } + + public YSQLExpression generateHavingClause() { + this.allowAggregateFunctions = true; + YSQLExpression expression = generateExpression(YSQLDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + + public YSQLExpression generateAggregate() { + return getAggregate(YSQLDataType.getRandomType()); + } + + private YSQLExpression getAggregate(YSQLDataType dataType) { + List aggregates = YSQLAggregate.YSQLAggregateFunction + .getAggregates(dataType); + YSQLAggregate.YSQLAggregateFunction agg = Randomly.fromList(aggregates); + return generateArgsForAggregate(dataType, agg); + } + + public YSQLAggregate generateArgsForAggregate(YSQLDataType dataType, YSQLAggregate.YSQLAggregateFunction agg) { + List types = agg.getTypes(dataType); + List args = new ArrayList<>(); + for (YSQLDataType argType : types) { + args.add(generateExpression(argType)); + } + return new YSQLAggregate(args, agg); + } + + public YSQLExpressionGenerator allowAggregates(boolean value) { + allowAggregateFunctions = value; + return this; + } + + @Override + public YSQLExpression generatePredicate() { + return generateExpression(YSQLDataType.BOOLEAN); + } + + @Override + public YSQLExpression negatePredicate(YSQLExpression predicate) { + return new YSQLPrefixOperation(predicate, YSQLPrefixOperation.PrefixOperator.NOT); + } + + @Override + public YSQLExpression isNull(YSQLExpression expr) { + return new YSQLPostfixOperation(expr, YSQLPostfixOperation.PostfixOperator.IS_NULL); + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, CAST, BETWEEN, IN_OPERATION, + SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON + } + + private enum RangeExpression { + BINARY_OP + } + + private enum TextExpression { + CAST, FUNCTION, CONCAT + } + + private enum BitExpression { + BINARY_OPERATION + } + + private enum IntExpression { + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + } + + public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState, String name, YSQLTables tables) { + List columns = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + YSQLSelect select = new YSQLSelect(); + select.setFromList(tables.getTables().stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(YSQLSelect.ForClause.getRandom()); + } + return new YSQLSelect.YSQLSubquery(select, name); + } + + @Override + public YSQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public YSQLExpression generateBooleanExpression() { + return generateExpression(YSQLDataType.BOOLEAN); + } + + @Override + public YSQLSelect generateSelect() { + return new YSQLSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLTable table = Randomly.fromList(tables); + tables.remove(table); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLSelect.YSQLSubquery subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(YSQLSelect select, YSQLExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + YSQLAggregate aggr = new YSQLAggregate(List.of(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)), + YSQLAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(Arrays.asList(allColumns)); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + select.setWhereClause(whereCondition); + } + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(YSQLSelect select, YSQLExpression whereCondition) { + YSQLCastOperation isTrue = new YSQLCastOperation(whereCondition, YSQLCompoundDataType.create(YSQLDataType.INT)); + YSQLPostfixText asText = new YSQLPostfixText(isTrue, " as count", null, YSQLDataType.INT); + select.setFetchColumns(Collections.singletonList(asText)); + select.setSelectType(YSQLSelect.SelectType.ALL); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBooleanWithRatherLowProbability()) { + return List.of(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)); + } + return columns.stream().map(c -> new YSQLColumnValue(c, null)).collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java new file mode 100644 index 000000000..5453746b2 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -0,0 +1,143 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.gen.AbstractIndexGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public class YSQLIndexGenerator extends AbstractIndexGenerator { + + private final YSQLGlobalState globalState; + + public YSQLIndexGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(YSQLGlobalState globalState) { + return new YSQLIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(Randomly.getBoolean()); + YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized + // views + String indexName = getNewIndexName(randomTable); + sb.append(indexName); + sb.append(" ON "); + if (Randomly.getBoolean()) { + sb.append("ONLY "); + } + sb.append(randomTable.getName()); + IndexType method; + if (Randomly.getBoolean()) { + sb.append(" USING "); + method = Randomly.fromOptions(IndexType.values()); + sb.append(method); + } else { + method = IndexType.BTREE; + } + + sb.append("("); + if (method == IndexType.HASH) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + if (Randomly.getBoolean()) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + sb.append("("); + YSQLExpression expression = YSQLExpressionGenerator.generateExpression(globalState, + randomTable.getColumns()); + sb.append(YSQLVisitor.asString(expression)); + sb.append(")"); + } + + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" "); + sb.append(globalState.getRandomOpclass()); + errors.add("does not accept"); + errors.add("does not exist for access method"); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + } + + sb.append(")"); + if (Randomly.getBoolean() && method != IndexType.HASH) { + sb.append(" INCLUDE("); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + } + if (Randomly.getBoolean()) { + YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()) + .setGlobalState(globalState).generateExpression(YSQLDataType.BOOLEAN); + appendWhereClause(YSQLVisitor.asString(expr)); + } + errors.add("already contains data"); // CONCURRENT INDEX failed + errors.add("You might need to add explicit type casts"); + errors.add("INDEX on column of type"); + errors.add("collations are not supported"); // TODO check + errors.add("because it has pending trigger events"); + errors.add("duplicate key value violates unique constraint"); + errors.add("could not determine which collation to use for"); + errors.add("index method \"gist\" not supported yet"); + errors.add("is duplicated"); + errors.add("already exists"); + errors.add("could not create unique index"); + errors.add("has no default operator class"); + errors.add("does not support"); + errors.add("cannot cast"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("invalid input syntax for"); + errors.add("must be type "); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("out of range"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("result of range difference would not be contiguous"); + errors.add("which is part of the partition key"); + YSQLErrors.addCommonExpressionErrors(errors); + } + + private static String getNewIndexName(YSQLTable randomTable) { + List indexes = randomTable.getIndexes(); + int indexI = 0; + while (true) { + String indexName = DBMSCommon.createIndexName(indexI++); + if (indexes.stream().noneMatch(i -> i.getIndexName().equals(indexName))) { + return indexName; + } + } + } + + public enum IndexType { + BTREE, HASH, GIST, GIN + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java new file mode 100644 index 000000000..c26459f31 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java @@ -0,0 +1,128 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public final class YSQLInsertGenerator { + + private YSQLInsertGenerator() { + } + + public static SQLQueryAdapter insert(YSQLGlobalState globalState) { + YSQLTable table = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("cannot insert into column"); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonInsertUpdateErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + errors.add("multiple assignments to same column"); + errors.add("violates foreign key constraint"); + errors.add("value too long for type character varying"); + errors.add("conflicting key value violates exclusion constraint"); + errors.add("violates not-null constraint"); + errors.add("current transaction is aborted"); + errors.add("bit string too long"); + errors.add("new row violates check option for view"); + errors.add("reached maximum value of sequence"); + errors.add("but expression is of type"); + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" OVERRIDING"); + sb.append(" "); + sb.append(Randomly.fromOptions("SYSTEM", "USER")); + sb.append(" VALUE"); + } + sb.append(" VALUES"); + + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + StringBuilder sbRowValue = new StringBuilder(); + sbRowValue.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sbRowValue.append(", "); + } + sbRowValue.append(YSQLVisitor.asString( + YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), columns.get(i).getType()))); + } + sbRowValue.append(")"); + + int n = (int) Randomly.getNotCachedInteger(100, 1000); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(sbRowValue); + } + } else { + int n = Randomly.smallNumber() + 1; + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + insertRow(globalState, sb, columns, n == 1); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" ON CONFLICT "); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(table.getRandomColumn().getName()); + sb.append(")"); + errors.add("there is no unique or exclusion constraint matching the ON CONFLICT specification"); + } + sb.append(" DO NOTHING"); + } + errors.add("duplicate key value violates unique constraint"); + errors.add("identity column defined as GENERATED ALWAYS"); + errors.add("out of range"); + errors.add("violates check constraint"); + errors.add("no partition of relation"); + errors.add("invalid input syntax"); + errors.add("division by zero"); + errors.add("violates foreign key constraint"); + errors.add("data type unknown"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private static void insertRow(YSQLGlobalState globalState, StringBuilder sb, List columns, + boolean canBeDefault) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + if (!Randomly.getBooleanWithSmallProbability() || !canBeDefault) { + YSQLExpression generateConstant; + if (Randomly.getBoolean()) { + generateConstant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), + columns.get(i).getType()); + } else { + generateConstant = new YSQLExpressionGenerator(globalState) + .generateExpression(columns.get(i).getType()); + } + sb.append(YSQLVisitor.asString(generateConstant)); + } else { + sb.append("DEFAULT"); + } + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java new file mode 100644 index 000000000..a6fb58d8e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLNotifyGenerator { + + private YSQLNotifyGenerator() { + } + + private static String getChannel() { + return Randomly.fromOptions("asdf", "test"); + } + + public static SQLQueryAdapter createNotify(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("NOTIFY "); + sb.append(getChannel()); + if (Randomly.getBoolean()) { + sb.append(", "); + sb.append("'"); + sb.append(globalState.getRandomly().getString().replace("'", "''")); + sb.append("'"); + } + return new SQLQueryAdapter(sb.toString()); + } + + public static SQLQueryAdapter createListen() { + String sb = "LISTEN " + getChannel(); + return new SQLQueryAdapter(sb); + } + + public static SQLQueryAdapter createUnlisten() { + StringBuilder sb = new StringBuilder(); + sb.append("UNLISTEN "); + if (Randomly.getBoolean()) { + sb.append(getChannel()); + } else { + sb.append("*"); + } + return new SQLQueryAdapter(sb.toString()); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java new file mode 100644 index 000000000..9821dcb7d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java @@ -0,0 +1,62 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.ForClause; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.SelectType; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; + +public final class YSQLRandomQueryGenerator { + + private YSQLRandomQueryGenerator() { + } + + public static YSQLSelect createRandomQuery(int nrColumns, YSQLGlobalState globalState) { + List columns = new ArrayList<>(); + YSQLTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < nrColumns; i++) { + columns.add(gen.generateExpression(0)); + } + YSQLSelect select = new YSQLSelect(); + select.setSelectType(SelectType.getRandom()); + if (select.getSelectOption() == SelectType.DISTINCT && Randomly.getBoolean()) { + select.setDistinctOnClause(gen.generateExpression(0)); + } + select.setFromList(tables.getTables().stream().map(t -> new YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return select; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java new file mode 100644 index 000000000..7526d8dc6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java @@ -0,0 +1,58 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; + +public final class YSQLReindexGenerator { + + private YSQLReindexGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("could not create unique index"); // CONCURRENT INDEX + StringBuilder sb = new StringBuilder(); + sb.append("REINDEX"); + // if (Randomly.getBoolean()) { + // sb.append(" VERBOSE"); + // } + sb.append(" "); + Scope scope = Randomly.fromOptions(Scope.values()); + switch (scope) { + case INDEX: + sb.append("INDEX "); + List indexes = globalState.getSchema().getRandomTable().getIndexes(); + if (indexes.isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(indexes.stream().map(YSQLIndex::getIndexName).collect(Collectors.joining())); + break; + case TABLE: + sb.append("TABLE "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + break; + case DATABASE: + sb.append("DATABASE "); + sb.append(globalState.getSchema().getDatabaseName()); + break; + default: + throw new AssertionError(scope); + } + errors.add("already contains data"); // FIXME bug report + errors.add("does not exist"); // internal index + errors.add("REINDEX is not yet implemented for partitioned indexes"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private enum Scope { + INDEX, TABLE, DATABASE + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java new file mode 100644 index 000000000..3d7104b5e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java @@ -0,0 +1,90 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLSequenceGenerator { + + private YSQLSequenceGenerator() { + } + + public static SQLQueryAdapter createSequence(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("TEMPORARY", "TEMP")); + } + sb.append(" SEQUENCE"); + // TODO keep track of sequences + sb.append(" IF NOT EXISTS"); + // TODO generate sequence names + sb.append(" seq"); + if (Randomly.getBoolean()) { + sb.append(" AS "); + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + } + if (Randomly.getBoolean()) { + sb.append(" INCREMENT"); + if (Randomly.getBoolean()) { + sb.append(" BY"); + } + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + errors.add("INCREMENT must not be zero"); + } + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" MINVALUE"); + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + } else { + sb.append(" NO MINVALUE"); + } + errors.add("must be less than MAXVALUE"); + } + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" MAXVALUE"); + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + } else { + sb.append(" NO MAXVALUE"); + } + errors.add("must be less than MAXVALUE"); + } + if (Randomly.getBoolean()) { + sb.append(" START"); + if (Randomly.getBoolean()) { + sb.append(" WITH"); + } + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + errors.add("cannot be less than MINVALUE"); + errors.add("cannot be greater than MAXVALUE"); + } + if (Randomly.getBoolean()) { + sb.append(" CACHE "); + sb.append(globalState.getRandomly().getPositiveIntegerNotNull()); + } + errors.add("is out of range"); + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" NO"); + } + sb.append(" CYCLE"); + } + if (Randomly.getBoolean()) { + sb.append(" OWNED BY "); + // if (Randomly.getBoolean()) { + sb.append("NONE"); + // } else { + // sb.append(s.getRandomTable().getRandomColumn().getFullQualifiedName()); + // } + } + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java new file mode 100644 index 000000000..3406ca3de --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java @@ -0,0 +1,196 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.function.Function; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLSetGenerator { + + private YSQLSetGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + ArrayList options = new ArrayList<>(Arrays.asList(ConfigurationOption.values())); + options.remove(ConfigurationOption.DEFAULT_WITH_OIDS); + ConfigurationOption option = Randomly.fromList(options); + sb.append("SET "); + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions("SESSION", "LOCAL")); + sb.append(" "); + } + sb.append(option.getOptionName()); + sb.append("="); + if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append(option.op.apply(globalState.getRandomly())); + } + // todo avoiding props that are not represented in YSQL + ExpectedErrors errors = new ExpectedErrors(); + errors.add("unrecognized configuration parameter"); + errors.add("cannot be changed"); + + return new SQLQueryAdapter(sb.toString(), errors); + } + + private enum ConfigurationOption { + // YUGABYTE + YB_DEBUG_REPORT_ERROR_STACKTRACE("yb_debug_report_error_stacktrace", + (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_CATCACHE_EVENTS("yb_debug_log_catcache_events", (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_INTERNAL_RESTARTS("yb_debug_log_internal_restarts", (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_DOCDB_REQUESTS("yb_debug_log_docdb_requests", (r) -> Randomly.fromOptions("false", "true")), + // YB_READ_FROM_FOLLOWERS("yb_read_from_followers", (r) -> Randomly.fromOptions("false", "true")), + YB_NON_DDL_TXN_FOR_SYS_TABLES_ALLOWED("yb_non_ddl_txn_for_sys_tables_allowed", + (r) -> Randomly.fromOptions("false", "true")), + YB_TRANSACTION_PRIORITY("yb_transaction_priority", + (r) -> Randomly.fromOptions(0, 0.1, 0.2, 0.3, 0.4, 1, 0.9, 0.8, 0.7, 0.6)), + YB_TRANSACTION_PRIORITY_LOWER_BOUND("yb_transaction_priority_lower_bound", + (r) -> Randomly.fromOptions(0, 0.1, 0.2, 0.3, 0.4)), + YB_TRANSACTION_PRIORITY_UPPER_BOUND("yb_transaction_priority_upper_bound", + (r) -> Randomly.fromOptions(1, 0.9, 0.8, 0.7, 0.6)), + YB_FORMAT_FUNCS_INCLUDE_YB_METADATA("yb_format_funcs_include_yb_metadata", + (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_GEOLOCATION_COSTING("yb_enable_geolocation_costing", (r) -> Randomly.fromOptions("false", "true")), + YB_BINARY_RESTORE("yb_binary_restore", (r) -> Randomly.fromOptions("false", "true")), + YB_TEST_SYSTEM_CATALOGS_CREATION("yb_test_system_catalogs_creation", + (r) -> Randomly.fromOptions("false", "true")), + YB_TEST_FAIL_NEXT_DDL("yb_test_fail_next_ddl", (r) -> Randomly.fromOptions("false", "true")), + YB_DISABLE_TRANSACTIONAL_WRITES("yb_disable_transactional_writes", + (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_OPTIMIZER_STATISTICS("yb_enable_optimizer_statistics", (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_EXPRESSION_PUSHDOWN("yb_enable_expression_pushdown", (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_UPSERT_MODE("yb_enable_upsert_mode", (r) -> Randomly.fromOptions("false", "true")), + YB_PLANNER_CUSTOM_PLAN_FOR_PARTITION_PRUNING("yb_planner_custom_plan_for_partition_pruning", + (r) -> Randomly.fromOptions("false", "true")), + YB_INDEX_STATE_FLAGS_UPDATE_DELAY("yb_index_state_flags_update_delay", + (r) -> Randomly.getNotCachedInteger(200, 1000)), + YB_TEST_PLANNER_CUSTOM_PLAN_THRESHOLD("yb_test_planner_custom_plan_threshold", + (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + // YSQL values + YSQL_UPGRADE_MODE("ysql_upgrade_mode", (r) -> Randomly.fromOptions("false", "true")), + YSQL_SESSION_MAX_BATCH_SIZE("ysql_session_max_batch_size", + (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + YSQL_MAX_IN_FLIGHT_OPS("ysql_max_in_flight_ops", (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + // https://www.postgresql.org/docs/11/runtime-config-wal.html + // This parameter can only be set at server start. + // WAL_LEVEL("wal_level", (r) -> Randomly.fromOptions("replica", "minimal", "logical")), + // FSYNC("fsync", (r) -> Randomly.fromOptions(1, 0)), + SYNCHRONOUS_COMMIT("synchronous_commit", + (r) -> Randomly.fromOptions("remote_apply", "remote_write", "local", "off")), + WAL_COMPRESSION("wal_compression", (r) -> Randomly.fromOptions(1, 0)), + // wal_buffer: server start + // wal_writer_delay: server start + // wal_writer_flush_after + COMMIT_DELAY("commit_delay", (r) -> r.getInteger(0, 100000)), + COMMIT_SIBLINGS("commit_siblings", (r) -> r.getInteger(0, 1000)), + // 19.5.2. Checkpoints + // checkpoint_timeout + // checkpoint_completion_target + // checkpoint_flush_after + // checkpoint_warning + // max_wal_size + // min_wal_size + // 19.5.3. Archiving + // archive_mode + // archive_command + // archive_timeout + // https://www.postgresql.org/docs/11/runtime-config-statistics.html + // 19.9.1. Query and Index Statistics Collector + TRACK_ACTIVITIES("track_activities", (r) -> Randomly.fromOptions(1, 0)), + // track_activity_query_size + TRACK_COUNTS("track_counts", (r) -> Randomly.fromOptions(1, 0)), + TRACK_IO_TIMING("track_io_timing", (r) -> Randomly.fromOptions(1, 0)), + TRACK_FUNCTIONS("track_functions", (r) -> Randomly.fromOptions("'none'", "'pl'", "'all'")), + // stats_temp_directory + // TODO 19.9.2. Statistics Monitoring + // https://www.postgresql.org/docs/11/runtime-config-autovacuum.html + // all can only be set at server-conf time + // 19.11. Client Connection Defaults + VACUUM_FREEZE_TABLE_AGE("vacuum_freeze_table_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), + VACUUM_FREEZE_MIN_AGE("vacuum_freeze_min_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), + VACUUM_MULTIXACT_FREEZE_TABLE_AGE("vacuum_multixact_freeze_table_age", + (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), + VACUUM_MULTIXACT_FREEZE_MIN_AGE("vacuum_multixact_freeze_min_age", + (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), + VACUUM_CLEANUP_INDEX_SCALE_FACTOR("vacuum_cleanup_index_scale_factor", + (r) -> Randomly.fromOptions(0.0, 0.0000001, 0.00001, 0.01, 0.1, 1, 10, 100, 100000, 10000000000.0)), + // TODO others + GIN_FUZZY_SEARCH_LIMIT("gin_fuzzy_search_limit", (r) -> r.getInteger(0, 2147483647)), + // 19.13. Version and Platform Compatibility + DEFAULT_WITH_OIDS("default_with_oids", (r) -> Randomly.fromOptions(0, 1)), + SYNCHRONIZED_SEQSCANS("synchronize_seqscans", (r) -> Randomly.fromOptions(0, 1)), + // https://www.postgresql.org/docs/devel/runtime-config-query.html + ENABLE_BITMAPSCAN("enable_bitmapscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_GATHERMERGE("enable_gathermerge", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_HASHJOIN("enable_hashjoin", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_INDEXSCAN("enable_indexscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_INDEXONLYSCAN("enable_indexonlyscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_MATERIAL("enable_material", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_MERGEJOIN("enable_mergejoin", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_NESTLOOP("enable_nestloop", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARALLEL_APPEND("enable_parallel_append", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARALLEL_HASH("enable_parallel_hash", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITION_PRUNING("enable_partition_pruning", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITIONWISE_JOIN("enable_partitionwise_join", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITIONWISE_AGGREGATE("enable_partitionwise_aggregate", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_SEGSCAN("enable_seqscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_SORT("enable_sort", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_TIDSCAN("enable_tidscan", (r) -> Randomly.fromOptions(1, 0)), + // 19.7.2. Planner Cost Constants (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS + SEQ_PAGE_COST("seq_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + RANDOM_PAGE_COST("random_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_TUPLE_COST("cpu_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_INDEX_TUPLE_COST("cpu_index_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_OPERATOR_COST("cpu_operator_cost", (r) -> Randomly.fromOptions(0d, 0.000001, 0.0025, 0.1, 1, 10, 10000)), + PARALLEL_SETUP_COST("parallel_setup_cost", (r) -> r.getLong(0, Long.MAX_VALUE)), + PARALLEL_TUPLE_COST("parallel_tuple_cost", (r) -> r.getLong(0, Long.MAX_VALUE)), + MIN_PARALLEL_TABLE_SCAN_SIZE("min_parallel_table_scan_size", (r) -> r.getInteger(0, 715827882)), + MIN_PARALLEL_INDEX_SCAN_SIZE("min_parallel_index_scan_size", (r) -> r.getInteger(0, 715827882)), + EFFECTIVE_CACHE_SIZE("effective_cache_size", (r) -> r.getInteger(1, 2147483647)), + JIT_ABOVE_COST("jit_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE - 1))), + JIT_INLINE_ABOVE_COST("jit_inline_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), + JIT_OPTIMIZE_ABOVE_COST("jit_optimize_above_cost", + (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), + // 19.7.3. Genetic Query Optimizer (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO + GEQO("geqo", (r) -> Randomly.fromOptions(1, 0)), + GEQO_THRESHOLD("geqo_threshold", (r) -> r.getInteger(2, 2147483647)), + GEQO_EFFORT("geqo_effort", (r) -> r.getInteger(1, 10)), + GEQO_POO_SIZE("geqo_pool_size", (r) -> r.getInteger(0, 2147483647)), + GEQO_GENERATIONS("geqo_generations", (r) -> r.getInteger(0, 2147483647)), + GEQO_SELECTION_BIAS("geqo_selection_bias", (r) -> Randomly.fromOptions(1.5, 1.8, 2.0)), + GEQO_SEED("geqo_seed", (r) -> Randomly.fromOptions(0, 0.5, 1)), + // 19.7.4. Other Planner Options (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER + DEFAULT_STATISTICS_TARGET("default_statistics_target", (r) -> r.getInteger(1, 10000)), + CONSTRAINT_EXCLUSION("constraint_exclusion", (r) -> Randomly.fromOptions("on", "off", "partition")), + CURSOR_TUPLE_FRACTION("cursor_tuple_fraction", + (r) -> Randomly.fromOptions(0.0, 0.1, 0.000001, 1, 0.5, 0.9999999)), + FROM_COLLAPSE_LIMIT("from_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), + JIT("jit", (r) -> Randomly.fromOptions(1, 0)), + JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), + PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), + FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")); + + private final String optionName; + private final Function op; + + ConfigurationOption(String optionName, Function op) { + this.optionName = optionName; + this.op = op; + } + + public String getOptionName() { + return optionName; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java new file mode 100644 index 000000000..3b212b00b --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java @@ -0,0 +1,74 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLStatisticsObject; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLStatisticsGenerator { + + private YSQLStatisticsGenerator() { + } + + public static SQLQueryAdapter insert(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE STATISTICS "); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO materialized view + if (randomTable.getColumns().size() < 2) { + throw new IgnoreMeException(); + } + sb.append(" "); + sb.append(getNewStatisticsName(randomTable)); + if (Randomly.getBoolean()) { + sb.append(" ("); + List statsSubset; + statsSubset = Randomly.nonEmptySubset("ndistinct", "dependencies", "mcv"); + sb.append(String.join(", ", statsSubset)); + sb.append(")"); + } + + List randomColumns = randomTable.getRandomNonEmptyColumnSubset( + globalState.getRandomly().getInteger(2, randomTable.getColumns().size())); + sb.append(" ON "); + sb.append(randomColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(" FROM "); + sb.append(randomTable.getName()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot have more than 8 columns in statistics"), + true); + } + + public static SQLQueryAdapter remove(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder("DROP STATISTICS "); + YSQLTable randomTable = globalState.getSchema().getRandomTable(); + List statistics = randomTable.getStatistics(); + if (statistics.isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(Randomly.fromList(statistics).getName()); + return new SQLQueryAdapter(sb.toString(), true); + } + + private static String getNewStatisticsName(YSQLTable randomTable) { + List statistics = randomTable.getStatistics(); + int i = 0; + while (true) { + String candidateName = "s" + i; + if (statistics.stream().noneMatch(stat -> stat.getName().contentEquals(candidateName))) { + return candidateName; + } + i++; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java new file mode 100644 index 000000000..37945c7ac --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java @@ -0,0 +1,248 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public class YSQLTableGenerator { + + protected final ExpectedErrors errors = new ExpectedErrors(); + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + private final List columnsToBeAdded = new ArrayList<>(); + private final YSQLTable table; + private final boolean generateOnlyKnown; + private final YSQLGlobalState globalState; + private boolean columnCanHavePrimaryKey; + private boolean columnHasPrimaryKey; + private boolean isTemporaryTable; + + public YSQLTableGenerator(String tableName, boolean generateOnlyKnown, YSQLGlobalState globalState) { + this.tableName = tableName; + this.generateOnlyKnown = generateOnlyKnown; + this.globalState = globalState; + table = new YSQLTable(tableName, columnsToBeAdded, null, null, null, false, false); + // YB catalog specific messages + errors.add("The catalog snapshot used for this transaction has been invalidated"); + + errors.add("PRIMARY KEY containing column of type"); + errors.add("specified value cannot be cast to type boolean for column"); + errors.add("already exists"); + errors.add("invalid input syntax for"); + errors.add("is not unique"); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("cannot create partitioned table as inheritance child"); + errors.add("cannot cast"); + errors.add("ERROR: functions in index expression must be marked IMMUTABLE"); + errors.add("functions in partition key expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not exist for access method"); + errors.add("does not accept data type"); + errors.add("but default expression is of type text"); + errors.add("has pseudo-type unknown"); + errors.add("no collation was derived for partition key column"); + errors.add("cannot set colocated true on a non-colocated database"); + errors.add("Cannot split table that does not have primary key"); + errors.add("inherits from generated column but specifies identity"); + errors.add("inherits from generated column but specifies default"); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonTableErrors(errors); + } + + public static SQLQueryAdapter generate(String tableName, boolean generateOnlyKnown, YSQLGlobalState globalState) { + return new YSQLTableGenerator(tableName, generateOnlyKnown, globalState).generate(); + } + + private SQLQueryAdapter generate() { + columnCanHavePrimaryKey = true; + sb.append("CREATE"); + if (Randomly.getBooleanWithSmallProbability()) { + sb.append(" "); + isTemporaryTable = true; + sb.append(Randomly.fromOptions("TEMPORARY", "TEMP")); + } + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + createStandard(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void createStandard() throws AssertionError { + sb.append("("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + String name = DBMSCommon.createColumnName(i); + createColumn(name); + } + if (Randomly.getBoolean()) { + errors.add("constraints on temporary tables may reference only temporary tables"); + errors.add("constraints on unlogged tables may reference only permanent or unlogged tables"); + errors.add("constraints on permanent tables may reference only permanent tables"); + errors.add("cannot be implemented"); + errors.add("there is no unique constraint matching given keys for referenced table"); + errors.add("cannot reference partitioned table"); + errors.add("unsupported ON COMMIT and foreign key combination"); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("option is not yet supported for hash partitioned tables"); + YSQLCommon.addTableConstraints(columnHasPrimaryKey, sb, table, globalState, errors); + } + sb.append(")"); + generatePartitionBy(); + YSQLCommon.generateWith(sb, globalState, errors, columnsToBeAdded, isTemporaryTable); + if (Randomly.getBoolean() && isTemporaryTable) { + sb.append(" ON COMMIT "); + // todo ON COMMIT DROP fails and it's known issue + // sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS", "DROP")); + sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS")); + sb.append(" "); + } + } + + private void createColumn(String name) throws AssertionError { + sb.append(name); + sb.append(" "); + YSQLDataType type = YSQLDataType.getRandomType(); + boolean serial = YSQLCommon.appendDataType(type, sb, true, generateOnlyKnown, globalState.getCollates()); + YSQLColumn c = new YSQLColumn(name, type); + c.setTable(table); + columnsToBeAdded.add(c); + sb.append(" "); + if (Randomly.getBoolean()) { + createColumnConstraint(type, serial); + } + } + + private void generatePartitionBy() { + if (Randomly.getBoolean()) { + return; + } + sb.append(" PARTITION BY "); + // TODO "RANGE", + String partitionOption = Randomly.fromOptions("RANGE", "LIST", "HASH"); + sb.append(partitionOption); + sb.append("("); + errors.add("unrecognized parameter"); + errors.add("cannot use constant expression"); + errors.add("unrecognized parameter"); + errors.add("unsupported PRIMARY KEY constraint with partition key definition"); + errors.add("which is part of the partition key."); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("does not accept data type"); + int n = partitionOption.contentEquals("LIST") ? 1 : Randomly.smallNumber() + 1; + YSQLErrors.addCommonExpressionErrors(errors); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("("); + YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded); + sb.append(YSQLVisitor.asString(expr)); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(globalState.getRandomOpclass()); + errors.add("does not exist for access method"); + } + } + sb.append(")"); + } + + private void createColumnConstraint(YSQLDataType type, boolean serial) { + List constraintSubset = Randomly.nonEmptySubset(ColumnConstraint.values()); + if (Randomly.getBoolean()) { + // make checks constraints less likely + constraintSubset.remove(ColumnConstraint.CHECK); + } + if (!columnCanHavePrimaryKey || columnHasPrimaryKey) { + constraintSubset.remove(ColumnConstraint.PRIMARY_KEY); + } + if (constraintSubset.contains(ColumnConstraint.GENERATED) + && constraintSubset.contains(ColumnConstraint.DEFAULT)) { + // otherwise: ERROR: both default and identity specified for column + constraintSubset.remove(Randomly.fromOptions(ColumnConstraint.GENERATED, ColumnConstraint.DEFAULT)); + } + if (constraintSubset.contains(ColumnConstraint.GENERATED) && type != YSQLDataType.INT) { + // otherwise: ERROR: identity column type must be smallint, integer, or bigint + constraintSubset.remove(ColumnConstraint.GENERATED); + } + if (serial) { + constraintSubset.remove(ColumnConstraint.GENERATED); + constraintSubset.remove(ColumnConstraint.DEFAULT); + constraintSubset.remove(ColumnConstraint.NULL_OR_NOT_NULL); + + } + for (ColumnConstraint c : constraintSubset) { + sb.append(" "); + switch (c) { + case NULL_OR_NOT_NULL: + sb.append(Randomly.fromOptions("NOT NULL", "NULL")); + errors.add("conflicting NULL/NOT NULL declarations"); + break; + case UNIQUE: + sb.append("UNIQUE"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY"); + columnHasPrimaryKey = true; + break; + case DEFAULT: + sb.append("DEFAULT"); + sb.append(" ("); + sb.append(YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, type))); + sb.append(")"); + errors.add("out of range"); + errors.add("is a generated column"); + break; + case CHECK: + sb.append("CHECK ("); + sb.append(YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded, + YSQLDataType.BOOLEAN))); + sb.append(")"); + errors.add("out of range"); + break; + case GENERATED: + sb.append("GENERATED "); + if (Randomly.getBoolean()) { + sb.append(" ALWAYS AS ("); + sb.append(YSQLVisitor + .asString(YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded, type))); + sb.append(") STORED"); + errors.add("A generated column cannot reference another generated column."); + errors.add("cannot use generated column in partition key"); + errors.add("generation expression is not immutable"); + errors.add("cannot use column reference in DEFAULT expression"); + } else { + sb.append(Randomly.fromOptions("ALWAYS", "BY DEFAULT")); + sb.append(" AS IDENTITY"); + } + break; + default: + throw new AssertionError(sb); + } + } + } + + private enum ColumnConstraint { + NULL_OR_NOT_NULL, UNIQUE, PRIMARY_KEY, DEFAULT, CHECK, GENERATED + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java new file mode 100644 index 000000000..8bc292348 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.concurrent.atomic.AtomicLong; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLTableGroupGenerator { + + // TODO rework + public static final AtomicLong UNIQUE_TABLEGROUP_COUNTER = new AtomicLong(1); + + private YSQLTableGroupGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE TABLEGROUP "); + String tableGroupName = "tg" + UNIQUE_TABLEGROUP_COUNTER.incrementAndGet(); + sb.append(tableGroupName); + errors.add("cannot use tablegroups in a colocated database"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java new file mode 100644 index 000000000..67780c016 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class YSQLTransactionGenerator { + + private YSQLTransactionGenerator() { + } + + public static SQLQueryAdapter executeBegin() { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("BEGIN"); + if (Randomly.getBoolean()) { + errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); + sb.append(" ISOLATION LEVEL "); + sb.append(Randomly.fromOptions("SERIALIZABLE", "REPEATABLE READ", "READ COMMITTED")); + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append(Randomly.fromOptions("READ WRITE", "READ ONLY")); + // } + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java new file mode 100644 index 000000000..449654d99 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java @@ -0,0 +1,29 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTable; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLTruncateGenerator { + + private YSQLTruncateGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("TRUNCATE"); + if (Randomly.getBoolean()) { + sb.append(" TABLE"); + } + sb.append(" "); + sb.append(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty().stream().map(AbstractTable::getName) + .collect(Collectors.joining(", "))); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors + .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java new file mode 100644 index 000000000..1a9e1de76 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -0,0 +1,81 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public final class YSQLUpdateGenerator extends AbstractUpdateGenerator { + + private final YSQLGlobalState globalState; + private YSQLTable randomTable; + + private YSQLUpdateGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", + " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable")); + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + return new YSQLUpdateGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + randomTable = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + sb.append("UPDATE "); + sb.append(randomTable.getName()); + sb.append(" SET "); + errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced + // table multiple times + errors.add("new row violates check option for view"); + YSQLErrors.addCommonInsertUpdateErrors(errors); + + updateColumns(columns); + errors.add("invalid input syntax for "); + errors.add("operator does not exist: text = boolean"); + errors.add("violates check constraint"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("but expression is of type"); + YSQLErrors.addCommonExpressionErrors(errors); + if (!Randomly.getBooleanWithSmallProbability()) { + YSQLExpression where = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), + YSQLDataType.BOOLEAN); + appendWhereClause(YSQLVisitor.asString(where)); + } + } + + @Override + protected void updateValue(YSQLColumn column) { + if (!Randomly.getBoolean()) { + YSQLExpression constant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(YSQLVisitor.asString(constant)); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append("("); + YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), + column.getType()); + // caused by casts + sb.append(YSQLVisitor.asString(expr)); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java new file mode 100644 index 000000000..9efe027b1 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java @@ -0,0 +1,16 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLVacuumGenerator { + + private YSQLVacuumGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + return new SQLQueryAdapter("VACUUM", ExpectedErrors.from("VACUUM cannot run inside a transaction block")); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java new file mode 100644 index 000000000..3005d49f8 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -0,0 +1,52 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; + +public final class YSQLViewGenerator { + + private YSQLViewGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + } else { + if (Randomly.getBoolean()) { + sb.append(" OR REPLACE"); + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" TEMP", " TEMPORARY")); + } + } + sb.append(" VIEW "); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); + sb.append("("); + int nrColumns = Randomly.smallNumber() + 1; + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(DBMSCommon.createColumnName(i)); + } + sb.append(")"); + sb.append(" AS ("); + YSQLSelect select = YSQLRandomQueryGenerator.createRandomQuery(nrColumns, globalState); + sb.append(YSQLVisitor.asString(select)); + sb.append(")"); + YSQLErrors.addGroupingErrors(errors); + YSQLErrors.addViewErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java new file mode 100644 index 000000000..e5ae3fe1b --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -0,0 +1,96 @@ +package sqlancer.yugabyte.ysql.oracle; + +import static sqlancer.yugabyte.ysql.YSQLProvider.DDL_LOCK; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; + +public class YSQLCatalog implements TestOracle { + protected final YSQLGlobalState state; + + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final Main.StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + + private final List dmlActions = Arrays.asList(YSQLProvider.Action.INSERT, + YSQLProvider.Action.UPDATE, YSQLProvider.Action.DELETE); + private final List catalogActions = Arrays.asList(YSQLProvider.Action.CREATE_VIEW, + YSQLProvider.Action.CREATE_SEQUENCE, YSQLProvider.Action.ALTER_TABLE, YSQLProvider.Action.SET_CONSTRAINTS, + YSQLProvider.Action.DISCARD, YSQLProvider.Action.DROP_INDEX, YSQLProvider.Action.COMMENT_ON, + YSQLProvider.Action.RESET_ROLE, YSQLProvider.Action.RESET); + private final List diskActions = Arrays.asList(YSQLProvider.Action.TRUNCATE, + YSQLProvider.Action.VACUUM); + + public YSQLCatalog(YSQLGlobalState globalState) { + this.state = globalState; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); + } + + private YSQLProvider.Action getRandomAction(List actions) { + return actions.get(state.getRandomly().getInteger(0, actions.size())); + } + + protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { + synchronized (DDL_LOCK) { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + // TODO concurrent DDLs may produce a lot of noise in test logs so its disabled right now + // added timeout to avoid possible catalog collisions + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new AssertionError(); + } + + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = YSQLTableGenerator.generate(tableName, true, globalState); + globalState.executeStatement(createTable); + globalState.getManager().incrementSelectQueryCount(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + } catch (IgnoreMeException e) { + // do nothing + } + } + } + } + + @Override + public void check() throws Exception { + // create table or evaluate catalog test + int seed = state.getRandomly().getInteger(1, 100); + if (seed > 95) { + createTables(state, 1); + } else { + YSQLProvider.Action randomAction; + + if (seed > 40) { + randomAction = getRandomAction(dmlActions); + } else if (seed > 10) { + randomAction = getRandomAction(catalogActions); + } else { + randomAction = getRandomAction(diskActions); + } + + state.executeStatement(randomAction.getQuery(state)); + } + state.getManager().incrementSelectQueryCount(); + } +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java new file mode 100644 index 000000000..8d0cea997 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -0,0 +1,82 @@ +package sqlancer.yugabyte.ysql.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.gen.YSQLRandomQueryGenerator; + +public class YSQLFuzzer implements TestOracle { + private final YSQLGlobalState globalState; + private final List testQueries; + private final ExpectedErrors errors = new ExpectedErrors(); + + public YSQLFuzzer(YSQLGlobalState globalState) { + this.globalState = globalState; + + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); + YSQLErrors.addGroupingErrors(errors); + YSQLErrors.addViewErrors(errors); + + // remove timeout error from scope + errors.add("canceling statement due to statement timeout"); + + // exclude nemesis exceptions + errors.add("terminating connection due to administrator command"); + errors.add("Java heap space"); + errors.add("Connection refused"); + errors.add("Connection to"); + + testQueries = new ArrayList<>(); + + testQueries.add(new SelectQuery()); + testQueries.add(new ActionQuery(YSQLProvider.Action.UPDATE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.DELETE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.INSERT)); + } + + @Override + public void check() throws Exception { + Query s = testQueries.get(globalState.getRandomly().getInteger(0, testQueries.size())); + globalState.executeStatement(s.getQuery(globalState, errors)); + globalState.getManager().incrementSelectQueryCount(); + } + + private static class Query { + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + throw new IllegalAccessException("Should be implemented"); + }; + } + + private static class ActionQuery extends Query { + private final YSQLProvider.Action action; + + ActionQuery(YSQLProvider.Action action) { + this.action = action; + } + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + return action.getQuery(state); + } + } + + private static class SelectQuery extends Query { + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + return new SQLQueryAdapter( + YSQLVisitor.asString(YSQLRandomQueryGenerator.createRandomQuery(Randomly.smallNumber() + 1, state)) + + ";", + errors); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..8f54aa698 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java @@ -0,0 +1,147 @@ +package sqlancer.yugabyte.ysql.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLRowValue; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public class YSQLPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public YSQLPivotedQuerySynthesisOracle(YSQLGlobalState globalState) throws SQLException { + super(globalState); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); + } + + /* + * Prevent name collisions by aliasing the column. + */ + private YSQLColumn getFetchValueAliasedColumn(YSQLColumn c) { + YSQLColumn aliasedColumn = new YSQLColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType()); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + private List generateGroupByClause(List columns, YSQLRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> YSQLColumnValue.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private YSQLConstant generateLimit() { + if (Randomly.getBoolean()) { + return YSQLConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private YSQLExpression generateOffset() { + if (Randomly.getBoolean()) { + return YSQLConstant.createIntConstant(0); + } else { + return null; + } + } + + private YSQLExpression generateRectifiedExpression(List columns, YSQLRowValue rw) { + YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(YSQLDataType.BOOLEAN); + YSQLExpression result; + if (expr.getExpectedValue().isNull()) { + result = YSQLPostfixOperation.create(expr, YSQLPostfixOperation.PostfixOperator.IS_NULL); + } else { + result = YSQLPostfixOperation.create(expr, expr.getExpectedValue().cast(YSQLDataType.BOOLEAN).asBoolean() + ? YSQLPostfixOperation.PostfixOperator.IS_TRUE : YSQLPostfixOperation.PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (YSQLColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + @Override + public SQLQueryAdapter getRectifiedQuery() throws SQLException { + YSQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + + YSQLSelect selectStatement = new YSQLSelect(); + selectStatement.setSelectType(Randomly.fromOptions(YSQLSelect.SelectType.values())); + List columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + fetchColumns = columns; + selectStatement.setFromList(randomFromTables.getTables().stream() + .map(t -> new YSQLSelect.YSQLFromTable(t, false)).collect(Collectors.toList())); + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new YSQLColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + YSQLExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + YSQLExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + YSQLExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBys(); + selectStatement.setOrderByClauses(orderBy); + return new SQLQueryAdapter(YSQLVisitor.asString(selectStatement)); + } + + @Override + protected String getExpectedValues(YSQLExpression expr) { + return YSQLVisitor.asExpectedValues(expr); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java new file mode 100644 index 000000000..3fdaf3f9a --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -0,0 +1,194 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.postgresql.util.PSQLException; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; +import sqlancer.yugabyte.ysql.ast.YSQLAlias; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation.PostfixOperator; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation.PrefixOperator; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; + +public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public YSQLTLPAggregateOracle(YSQLGlobalState state) { + super(state); + YSQLErrors.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() throws SQLException { + YSQLAggregateFunction aggregateFunction = Randomly.fromOptions(YSQLAggregateFunction.MAX, + YSQLAggregateFunction.MIN, YSQLAggregateFunction.SUM, YSQLAggregateFunction.BIT_AND, + YSQLAggregateFunction.BIT_OR, YSQLAggregateFunction.BOOL_AND, YSQLAggregateFunction.BOOL_OR, + YSQLAggregateFunction.COUNT); + YSQLAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), + aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + originalQuery = YSQLVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, + secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + private String createMetamorphicUnionQuery(YSQLSelect select, YSQLAggregate aggregate, List from) { + String metamorphicQuery; + YSQLExpression whereClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLExpression negatedClause = new YSQLPrefixOperation(whereClause, PrefixOperator.NOT); + YSQLExpression notNullClause = new YSQLPostfixOperation(whereClause, PostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + YSQLSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); + YSQLSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); + YSQLSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += YSQLVisitor.asString(leftSelect) + " UNION ALL " + YSQLVisitor.asString(middleSelect) + + " UNION ALL " + YSQLVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + } catch (PSQLException e) { + throw new AssertionError(queryString, e); + } + return resultString; + } + + private List mapped(YSQLAggregate aggregate) { + switch (aggregate.getFunction()) { + case SUM: + case COUNT: + case BIT_AND: + case BIT_OR: + case BOOL_AND: + case BOOL_OR: + case MAX: + case MIN: + return aliasArgs(Arrays.asList(aggregate)); + // case AVG: + //// List arg = Arrays.asList(new + // YSQLCast(aggregate.getExpr().get(0), + // YSQLDataType.DECIMAL.get())); + // YSQLAggregate sum = new YSQLAggregate(YSQLAggregateFunction.SUM, + // aggregate.getExpr()); + // YSQLCast count = new YSQLCast( + // new YSQLAggregate(YSQLAggregateFunction.COUNT, aggregate.getExpr()), + // YSQLDataType.DECIMAL.get()); + //// YSQLBinaryArithmeticOperation avg = new + // YSQLBinaryArithmeticOperation(sum, count, + // YSQLBinaryArithmeticOperator.DIV); + // return aliasArgs(Arrays.asList(sum, count)); + default: + throw new AssertionError(aggregate.getFunction()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (YSQLExpression expr : originalAggregateArgs) { + args.add(new YSQLAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(YSQLAggregate aggregate) { + switch (aggregate.getFunction()) { + // case AVG: + // return "SUM(agg0::DECIMAL)/SUM(agg1)::DECIMAL"; + case COUNT: + return YSQLAggregateFunction.SUM + "(agg0)"; + default: + return aggregate.getFunction().toString() + "(agg0)"; + } + } + + private YSQLSelect getSelect(List aggregates, List from, YSQLExpression whereClause, + List joinList) { + YSQLSelect leftSelect = new YSQLSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinClauses(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java new file mode 100644 index 000000000..57fe246e3 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -0,0 +1,106 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + protected YSQLSchema s; + protected YSQLTables targetTables; + protected YSQLExpressionGenerator gen; + protected YSQLSelect select; + + public YSQLTLPBase(YSQLGlobalState state) { + super(state); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(state, targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + public static List getJoinStatements(YSQLGlobalState globalState, List columns, + List tables) { + List joinStatements = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLTable table = Randomly.fromList(tables); + tables.remove(table); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLSelect.YSQLSubquery subquery = YSQLExpressionGenerator.createSubquery(globalState, + String.format("sub%d", i), subqueryTables); + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream() + .map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + gen = new YSQLExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new YSQLSelect(); + select.setFetchColumns(generateFetchColumns()); + select.setFromList(tableList); + select.setWhereClause(null); + select.setJoinClauses(joins); + if (Randomly.getBoolean()) { + select.setForClause(YSQLSelect.ForClause.getRandom()); + } + } + + List generateFetchColumns() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(targetTables.getColumns()); + for (YSQLColumn c : targetColumns) { + fetchColumns.add(new YSQLColumnValue(c, null)); + } + return fetchColumns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java new file mode 100644 index 000000000..32df70b11 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java @@ -0,0 +1,66 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public class YSQLTLPHavingOracle extends YSQLTLPBase { + + public YSQLTLPHavingOracle(YSQLGlobalState state) { + super(state); + YSQLErrors.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + havingCheck(); + } + + @Override + List generateFetchColumns() { + List expressions = gen.allowAggregates(true).generateExpressions(Randomly.smallNumber() + 1); + gen.allowAggregates(false); + return expressions; + } + + protected void havingCheck() throws SQLException { + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(YSQLDataType.BOOLEAN)); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = YSQLVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + select.setHavingClause(predicate); + String firstQueryString = YSQLVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = YSQLVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = YSQLVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + protected YSQLExpression generatePredicate() { + return gen.generateHavingClause(); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java new file mode 100644 index 000000000..265586586 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.yugabyte.ysql.YSQLErrors; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public class YSQLTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; + + public YSQLTLPWhereOracle(YSQLGlobalState state) { + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) + .with(YSQLErrors.getCommonFetchErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); + } + + @Override + public void check() throws SQLException { + oracle.check(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } +} diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java new file mode 100644 index 000000000..f3a2ee2ba --- /dev/null +++ b/test/sqlancer/TestCommonSchema.java @@ -0,0 +1,186 @@ +package sqlancer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestCommonSchema { + static class TestTable extends AbstractTable> { + TestTable(String name, List columns, List indexes, boolean isView) { + super(name, columns, indexes, isView); + } + + @Override + public long getNrRows(GlobalState globalState) { + return 0; + } + } + + static class TestTableColumn extends AbstractTableColumn { + TestTableColumn(String name, TestTable table, String type) { + super(name, table, type); + } + } + + static class TestSchema extends AbstractSchema, TestTable> { + TestSchema(List tables) { + super(tables); + } + } + + static class TestTables extends AbstractTables { + TestTables(List tables) { + super(tables); + } + } + + static class TestIndex extends TableIndex { + TestIndex(String name) { + super(name); + } + } + + private TestTable createTestTable(String name, List indexes, boolean isView, String... columns) { + List cols = Arrays.stream(columns).map(col -> new TestTableColumn(col, null, "VARCHAR")) + .collect(Collectors.toList()); + return new TestTable(name, cols, indexes, isView); + } + + private TestTableColumn createTestColumn(String name, TestTable table, String type) { + return new TestTableColumn(name, table, type); + } + + private TestSchema createTestSchema(TestTable... tables) { + return new TestSchema(Arrays.asList(tables)); + } + + private TestTables createTestTables(TestTable... tables) { + return new TestTables(new ArrayList(Arrays.asList(tables))); + } + + @Test + void testColumnManagement() { + TestTable table = createTestTable("products", Collections.emptyList(), false, "sku", "price"); + List columnNames = table.getColumns().stream().map(TestTableColumn::getName) + .collect(Collectors.toList()); + List columnTypes = table.getColumns().stream().map(TestTableColumn::getType) + .collect(Collectors.toList()); + TestTableColumn randomCol = table.getRandomColumn(); + + assertTrue(columnNames.containsAll(Set.of("sku", "price"))); + assertTrue(columnTypes.containsAll(Set.of("VARCHAR"))); + assertTrue(table.getColumns().contains(randomCol)); + } + + @Test + void testIndexManagement() { + TestIndex idx1 = new TestIndex("idx_sku"); + TestIndex idx2 = new TestIndex("idx_price"); + TestTable table = createTestTable("products", Arrays.asList(idx1, idx2), false, "sku", "price"); + TableIndex randomIndex = table.getRandomIndex(); + + assertTrue(table.hasIndexes()); + assertEquals(2, table.getIndexes().size()); + assertTrue(table.getIndexes().contains(randomIndex)); + } + + @Test + void testViewManagement() { + TestTable view1 = createTestTable("v1", Collections.emptyList(), true, "col1"); + TestTable view2 = createTestTable("v2", Collections.emptyList(), true, "col2"); + TestTable table = createTestTable("t1", Collections.emptyList(), false, "col3"); + TestSchema schema = createTestSchema(view1, view2, table); + + assertAll(() -> assertEquals(2, schema.getViews().size(), "Should detect 2 views"), + () -> assertEquals(1, schema.getDatabaseTablesWithoutViews().size(), "Should detect 1 normal table"), + () -> assertEquals("t1", schema.getDatabaseTablesWithoutViews().get(0).getName())); + } + + @Test + void testFreeColumnNameGeneration() { + TestTable table = createTestTable("users", Collections.emptyList(), false, "id", "name"); + Set generatedNames = new HashSet<>(); + + for (int i = 0; i < 100; i++) { + String newName = table.getFreeColumnName(); + assertTrue(generatedNames.add(newName), "Duplicate: " + newName); + + List newColumns = new ArrayList<>(table.getColumns()); + newColumns.add(new TestTableColumn(newName, table, "TEXT")); + table = new TestTable(table.getName(), newColumns, table.getIndexes(), table.isView()); + } + } + + @Test + void testObjectComparison() { + TestTable tableA = createTestTable("A", Collections.emptyList(), false, "x", "y"); + TestTable tableB = createTestTable("B", Collections.emptyList(), false, "b"); + + TestTableColumn colA1 = new TestTableColumn("x", tableA, "INT"); + TestTableColumn colA2 = new TestTableColumn("y", tableA, "INT"); + TestTableColumn colB1 = new TestTableColumn("b", tableB, "TEXT"); + + assertAll(() -> assertTrue(colA1.compareTo(colA2) < 0, "Columns should be ordered by name"), + () -> assertTrue(colA1.compareTo(colB1) < 0, "Columns should be ordered by name"), + () -> assertTrue(tableA.compareTo(tableB) > 0, "Tables should be ordered reverse-alphabetically"), + () -> assertEquals(0, tableA.compareTo(tableA), "Same table should be equal")); + } + + @Test + void testEquality() { + TestTable table1 = createTestTable("t1", Collections.emptyList(), false, "id"); + TestTable table2 = createTestTable("t2", Collections.emptyList(), false, "id"); + + TestTableColumn col1 = new TestTableColumn("id", table1, "INT"); + TestTableColumn col2 = new TestTableColumn("id", table1, "INT"); + TestTableColumn col3 = new TestTableColumn("id", table2, "INT"); + TestTableColumn col4 = new TestTableColumn("name", table1, "TEXT"); + + assertAll(() -> assertEquals(col1, col2, "Same table/column should be equal"), + () -> assertNotEquals(col1, col3, "Different tables should not be equal"), + () -> assertNotEquals(col1, col4, "Different columns should not be equal"), + () -> assertNotEquals(col1, "invalid_object", "Different types should not be equal")); + } + + @Test + void testBoundaryConditions() { + String longName = "a".repeat(256); + TestTableColumn longCol = new TestTableColumn(longName, null, "TEXT"); + assertEquals(longName, longCol.getName()); + + TestTableColumn col2 = createTestColumn("orphan", null, "UNKNOWN"); + assertEquals("orphan", col2.getFullQualifiedName()); + assertNull(col2.getTable()); + } + + @Test + void testTablesManagement() { + TestTable table1 = createTestTable("t1", Collections.emptyList(), false, "col1"); + TestTable table2 = createTestTable("t2", Collections.emptyList(), false, "col2"); + TestTable table3 = createTestTable("t3", Collections.emptyList(), false, "col3"); + + TestTables tables = createTestTables(table1, table2, table3); + assertEquals(3, tables.getSize(), "Should detect 3 tables"); + assertEquals(3, tables.getColumns().size(), "Should detect 3 columns"); + assertTrue(tables.isContained(table3), "Table3 shoule be contained"); + + TestTable table4 = createTestTable("t4", Collections.emptyList(), false, "col4"); + tables.addTable(table4); + assertEquals(4, tables.getSize(), "Should detect 4 tables"); + assertEquals(4, tables.getColumns().size(), "Should detect 4 columns"); + assertTrue(tables.isContained(table4), "Table4 should be contained"); + + tables.removeTable(table4); + assertEquals(3, tables.getSize(), "Should detect 3 tables"); + assertEquals(3, tables.getColumns().size(), "Should detect 3 columns"); + assertTrue(!tables.isContained(table4), "Table4 should not be contained"); + } +} diff --git a/test/sqlancer/TestComparatorHelper.java b/test/sqlancer/TestComparatorHelper.java new file mode 100644 index 000000000..e8b06388b --- /dev/null +++ b/test/sqlancer/TestComparatorHelper.java @@ -0,0 +1,72 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.h2.H2Options; +import sqlancer.h2.H2Schema; + +public class TestComparatorHelper { + // TODO: Implement tests for the other ComparatorHelper methods + + // TODO: create test state that not depends on specific database + final SQLGlobalState state = new SQLGlobalState() { + + @Override + protected H2Schema readSchema() throws SQLException { + return H2Schema.fromConnection(getConnection(), getDatabaseName()); + } + + @Override + public MainOptions getOptions() { + return new MainOptions(); + } + }; + + @Test + public void testAssumeResultSetsAreEqualWithEqualSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "c"); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); + + } + + @Test + public void testAssumeResultSetsAreEqualWithUnequalLengthSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "c", "d", "g"); + // NullPointerException is raised instead of AssertionError because state is null and the state.getState()... + // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions + // is raised + assertThrowsExactly(NullPointerException.class, () -> { + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); + }); + } + + @Test + public void testAssumeResultSetsAreEqualWithUnequalValueSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "d"); + // NullPointerException is raised instead of AssertionError because state is null and the state.getState()... + // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions + // is raised + assertThrowsExactly(NullPointerException.class, () -> { + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); + }); + } + + @Test + public void testAssumeResultSetsAreEqualWithCanonicalizationRule() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "d"); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state, (String s) -> { + return s.equals("d") ? "c" : s; + }); + } + +} diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java new file mode 100644 index 000000000..2a3b1a67e --- /dev/null +++ b/test/sqlancer/TestExpectedErrors.java @@ -0,0 +1,159 @@ +package sqlancer; + +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.common.query.ExpectedErrors; + +public class TestExpectedErrors { + + @Test + public void testEmpty() { + ExpectedErrors errors = new ExpectedErrors(); + assertFalse(errors.errorIsExpected("a")); + } + + @Test + public void testStringSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("a"); + errors.add("b"); + errors.add("c"); + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + } + + @Test + public void testStringsSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAll(List.of("a", "b", "c")); + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + } + + @Test + public void testRegexSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegex(Pattern.compile("a\\d")); + errors.addRegex(Pattern.compile("b\\D")); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + } + + @Test + public void testRegexesSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAllRegexes(List.of(Pattern.compile("a\\d"), Pattern.compile("b\\D"))); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } + + @Test + public void testRegexStringSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegexString("a\\d"); + errors.addRegexString("b\\D"); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + } + + @Test + public void testRegexStrings() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAllRegexStrings(List.of("a\\d", "b\\D")); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } + + @Test + public void testStringRealistic() { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("violated"); + assertTrue(errors.errorIsExpected("UNIQUE constraint was violated!")); + assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); + } + + @Test + public void testRegexRealistic() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegex(Pattern.compile(".violated.")); + assertTrue(errors.errorIsExpected("UNIQUE constraint was violated!")); + assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); + } + + @Test + public void testBuilder() { + ExpectedErrors errors = ExpectedErrors.newErrors().with("a", "b", "c").build(); + + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + errors = ExpectedErrors.newErrors().with(List.of("a", "b", "c")).build(); + + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + errors = ExpectedErrors.newErrors().withRegexString("a\\d", "b\\D").with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().withRegexString(List.of("a\\d", "b\\D")).with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().withRegex(Pattern.compile("a\\d"), Pattern.compile("b\\D")).with("c") + .build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().withRegex(List.of(Pattern.compile("a\\d"), Pattern.compile("b\\D"))) + .with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } +} diff --git a/test/sqlancer/TestLoggableFactory.java b/test/sqlancer/TestLoggableFactory.java new file mode 100644 index 000000000..2679cf740 --- /dev/null +++ b/test/sqlancer/TestLoggableFactory.java @@ -0,0 +1,17 @@ +package sqlancer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.log.SQLLoggableFactory; +import sqlancer.common.query.SQLQueryAdapter; + +public class TestLoggableFactory { + + @Test + public void testLogCreateTable() { + String query = "CREATE TABLE t1 (c1 INT)"; + SQLLoggableFactory logger = new SQLLoggableFactory(); + SQLQueryAdapter queryAdapter = logger.getQueryForStateToReproduce(query); + assert (queryAdapter.couldAffectSchema()); + } + +} diff --git a/test/sqlancer/TestParameterFormat.java b/test/sqlancer/TestParameterFormat.java index fbdd3ad81..f9d89545a 100644 --- a/test/sqlancer/TestParameterFormat.java +++ b/test/sqlancer/TestParameterFormat.java @@ -17,11 +17,11 @@ */ public class TestParameterFormat { - private final static String OPTION_REGEX = "--[a-z0-9-]*"; + private final static String OPTION_REGEX = "(-)?-[a-z0-9-]*"; @Test public void testOptionFormat() throws Exception { - List> providers = Main.getDBMSProviders(); + List> providers = Main.getDBMSProviders(); MainOptions options = new MainOptions(); Builder commandBuilder = JCommander.newBuilder().addObject(options); List parameterDescriptions = new ArrayList<>(); @@ -38,7 +38,10 @@ public void testOptionFormat() throws Exception { parameterDescriptions.addAll(command.getParameters()); } for (ParameterDescription parameter : parameterDescriptions) { - assertTrue(Pattern.matches(OPTION_REGEX, parameter.getNames()), parameter.getNames()); + String[] names = parameter.getNames().split(", "); + for (String name : names) { + assertTrue(Pattern.matches(OPTION_REGEX, name), name); + } } } diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index 6be6b1bd4..8fbd97790 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -38,6 +38,7 @@ public void testSubset() { boolean encounteredStrictSubsetNonEmpty = false; Integer[] options = { 1, 2, 3 }; List optionList = new ArrayList<>(Arrays.asList(options)); + int i = 0; do { List subset = Randomly.subset(optionList); assertEquals(optionList.size(), 3); // check that the original set hasn't been modified @@ -49,7 +50,13 @@ public void testSubset() { } else { encounteredStrictSubsetNonEmpty = true; } - } while (!encounteredEmptySubset || !encounteredOriginalSet || !encounteredStrictSubsetNonEmpty); + } while (!encounteredEmptySubset || !encounteredOriginalSet || !encounteredStrictSubsetNonEmpty + || i++ < NR_MIN_RUNS); + + assertTrue(encounteredEmptySubset, "Empty subset was not encountered"); + assertTrue(encounteredOriginalSet, "Original set was not encountered"); + assertTrue(encounteredStrictSubsetNonEmpty, "Strict subset was not encountered"); + } @Test @@ -75,6 +82,10 @@ public void testString() { } } while (!encounteredInteger || !encounteredAscii || !encounteredNonAscii || !encounteredSpace || i++ < NR_MIN_RUNS); + assertTrue(encounteredInteger, "Integer was not encountered"); + assertTrue(encounteredAscii, "Ascii was not encountered"); + assertTrue(encounteredNonAscii, "Non ascii was not encountered"); + assertTrue(encounteredSpace, "Space was not encountered"); } @Test // TODO: also generate and check for NaN @@ -84,6 +95,7 @@ public void testDouble() { boolean encounteredPositive = false; boolean encounteredNegative = false; boolean encounteredInfinity = false; + int i = 0; do { double doubleVal = r.getDouble(); if (doubleVal == 0) { @@ -97,7 +109,12 @@ public void testDouble() { } else { fail(String.valueOf(doubleVal)); } - } while (!encounteredZero || !encounteredPositive || !encounteredNegative || !encounteredInfinity); + } while (!encounteredZero || !encounteredPositive || !encounteredNegative || !encounteredInfinity + || i++ < NR_MIN_RUNS); + assertTrue(encounteredZero, "Zero was not encountered"); + assertTrue(encounteredPositive, "Positive was not encountered"); + assertTrue(encounteredNegative, "Negative was not encountered"); + assertTrue(encounteredInfinity, "Infinity was not encountered"); } @Test @@ -123,6 +140,8 @@ public void testNonZeroInteger() { encounteredNegative = true; } } while (!encounteredPositive || !encounteredNegative || i++ < NR_MIN_RUNS); + assertTrue(encounteredPositive, "Positive integer was not encountered"); + assertTrue(encounteredNegative, "Negative integer was not encountered"); } @Test @@ -140,6 +159,8 @@ public void testPositiveInteger() { encounteredMaxValue = true; } } while (!encounteredZero || !encounteredMaxValue || i++ < NR_MIN_RUNS); + assertTrue(encounteredZero, "Zero was not encountered"); + assertTrue(encounteredMaxValue, "Max value was not encountered"); } @Test @@ -159,6 +180,9 @@ public void testBytes() { encounteredMax = true; } } while (!encounteredAllZeroes || !encounteredMax || !encounteredZeroLength || i++ < NR_MIN_RUNS); + assertTrue(encounteredAllZeroes, "All zeroes were not encountered"); + assertTrue(encounteredMax, "Max value was not encountered"); + assertTrue(encounteredZeroLength, "Zero length was not encountered"); } @Test @@ -234,4 +258,63 @@ private List getRandomValueList(Randomly r) { return values; } + @Test + public void testGetPercentage() { + for (int i = 0; i < NR_MIN_RUNS; i++) { + double percentage = Randomly.getPercentage(); + assertTrue(percentage >= 0.0); + assertTrue(percentage <= 1.0); + } + } + + @Test + public void testGetChar() { + Randomly r = new Randomly(); + boolean encounteredAlphabetic = false; + boolean encounteredNumeric = false; + boolean encounteredSpecial = false; + int i = 0; + do { + String c = r.getChar(); + assertEquals(1, c.length()); + if (Character.isAlphabetic(c.charAt(0))) { + encounteredAlphabetic = true; + } else if (Character.isDigit(c.charAt(0))) { + encounteredNumeric = true; + } else { + encounteredSpecial = true; + } + } while (!encounteredAlphabetic || !encounteredNumeric || !encounteredSpecial || i++ < NR_MIN_RUNS); + assertTrue(encounteredAlphabetic, "Never encounter an alphabetic character."); + assertTrue(encounteredNumeric, "Never encounter a numeric character."); + assertTrue(encounteredSpecial, "Never encounter a special character."); + } + + @Test + public void testGetAlphabeticChar() { + Randomly r = new Randomly(); + for (int i = 0; i < NR_MIN_RUNS; i++) { + String c = r.getAlphabeticChar(); + assertEquals(1, c.length()); + assertTrue(Character.isAlphabetic(c.charAt(0))); + } + } + + @Test + public void testGetBooleanWithSmallProbability() { + int trueCount = 0; + int totalRuns = NR_MIN_RUNS; + + for (int i = 0; i < totalRuns; i++) { + if (Randomly.getBooleanWithSmallProbability()) { + trueCount++; + } + } + + double trueRatio = (double) trueCount / totalRuns; + assertTrue(trueRatio > 0.005); + assertTrue(trueRatio < 0.015); + + } + } diff --git a/test/sqlancer/TestStateToReproduce.java b/test/sqlancer/TestStateToReproduce.java new file mode 100644 index 000000000..6fcdc22f1 --- /dev/null +++ b/test/sqlancer/TestStateToReproduce.java @@ -0,0 +1,109 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3Provider; + +public class TestStateToReproduce { + + @TempDir + Path tempDir; + + @Test + public void testBasicFields() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_db", provider); + state.databaseVersion = "3.36.0"; + state.seedValue = 12345L; + state.exception = "Test exception message"; + + Path file = tempDir.resolve("test_basic.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + assertEquals(state.getDatabaseName(), result.getDatabaseName()); + assertEquals(state.getDatabaseVersion(), result.getDatabaseVersion()); + assertEquals(state.getSeedValue(), result.getSeedValue()); + assertEquals(state.getException(), result.getException()); + } + + @Test + public void testStatements() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_statements", provider); + List> statements = new ArrayList<>(); + + ExpectedErrors errors1 = new ExpectedErrors(); + errors1.add("syntax error"); + errors1.add("table already exists"); + statements.add(new SQLQueryAdapter("CREATE TABLE test (id INTEGER);", errors1)); + + ExpectedErrors errors2 = new ExpectedErrors(); + errors2.add("constraint failed"); + statements.add(new SQLQueryAdapter("INSERT INTO test VALUES (1);", errors2)); + + statements.add(new SQLQueryAdapter("SELECT * FROM test;", new ExpectedErrors())); + state.setStatements(statements); + + Path file = tempDir.resolve("test_statements.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + List> resultStatements = result.getStatements(); + assertEquals(3, resultStatements.size()); + + Query q1 = resultStatements.get(0); + Query q2 = resultStatements.get(1); + Query q3 = resultStatements.get(2); + + assertEquals("CREATE TABLE test (id INTEGER);", q1.getLogString()); + assertEquals("INSERT INTO test VALUES (1);", q2.getLogString()); + assertEquals("SELECT * FROM test;", q3.getLogString()); + + ExpectedErrors e1 = q1.getExpectedErrors(); + ExpectedErrors e2 = q2.getExpectedErrors(); + ExpectedErrors e3 = q3.getExpectedErrors(); + + assertTrue(e1.errorIsExpected("syntax error")); + assertTrue(e1.errorIsExpected("table already exists")); + assertFalse(e1.errorIsExpected("constraint failed")); + + assertFalse(e2.errorIsExpected("syntax error")); + assertTrue(e2.errorIsExpected("constraint failed")); + + assertFalse(e3.errorIsExpected("syntax error")); + assertFalse(e3.errorIsExpected("constraint failed")); + } + + @Test + public void testDatabaseProvider() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_provider", provider); + state.logStatement("CREATE TABLE test (id INTEGER);"); + + Path file = tempDir.resolve("test_provider.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + // Verify databaseProvider is correctly deserialized + assertEquals("sqlite3", result.getDatabaseProvider().getDBMSName()); + + // Verify databaseProvider functionality by testing logStatement + result.logStatement("INSERT INTO test VALUES (1);"); + assertEquals(2, result.getStatements().size()); + assertEquals("INSERT INTO test VALUES (1);", result.getStatements().get(1).getLogString()); + } +} \ No newline at end of file diff --git a/test/sqlancer/TestUsageNamingConvention.java b/test/sqlancer/TestUsageNamingConvention.java new file mode 100644 index 000000000..94f5daba0 --- /dev/null +++ b/test/sqlancer/TestUsageNamingConvention.java @@ -0,0 +1,34 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.JCommander.Builder; +import com.beust.jcommander.Parameters; + +public class TestUsageNamingConvention { + + @Test + void testNonEmptyDescription() { + List> providers = Main.getDBMSProviders(); + MainOptions options = new MainOptions(); + Builder commandBuilder = JCommander.newBuilder().addObject(options); + for (DatabaseProvider provider : providers) { + String name = provider.getDBMSName(); + if (!name.toLowerCase().equals(name)) { + throw new AssertionError(name + " should be in lowercase!"); + } + commandBuilder.addCommand(provider.getDBMSName(), provider.getOptionClass()); + Parameters param = provider.getOptionClass().getAnnotation(Parameters.class); + assertNotEquals(null, param, provider.getOptionClass().toString()); + String databaseDescription = param.commandDescription(); + assertNotEquals(null, databaseDescription, "description cannot be empty " + provider.getOptionClass()); + assertNotEquals("", databaseDescription, "description cannot be empty " + provider.getOptionClass()); + } + } + +} diff --git a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java index afbe76227..97ca1e282 100644 --- a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java +++ b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java @@ -1,130 +1,130 @@ package sqlancer.clickhouse.ast; import org.junit.jupiter.api.Test; -import ru.yandex.clickhouse.domain.ClickHouseDataType; import java.util.Arrays; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import com.clickhouse.client.ClickHouseDataType; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; + +import static org.junit.jupiter.api.Assertions.assertEquals; class ClickHouseBinaryComparisonOperationTest { @Test void getExpectedValueTrueEqualsTrue() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant equals = trueConst.applyEquals(ClickHouseConstant.createTrue()); - assertEquals(equals.asInt(), 1); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant equals = trueConst.applyEquals(ClickHouseCreateConstant.createTrue()); + assertEquals(true, equals.asBooleanNotNull()); } @Test void getExpectedValueTrueNotEqualsFalse() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); - ClickHouseConstant equals = trueConst.applyEquals(ClickHouseConstant.createFalse()); - ClickHouseConstant equalsFalse = falseConst.applyEquals(ClickHouseConstant.createTrue()); - assertEquals(equals.asInt(), 0); - assertEquals(equalsFalse.asInt(), 0); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); + ClickHouseConstant equals = trueConst.applyEquals(ClickHouseCreateConstant.createFalse()); + ClickHouseConstant equalsFalse = falseConst.applyEquals(ClickHouseCreateConstant.createTrue()); + assertEquals(false, equals.asBooleanNotNull()); + assertEquals(false, equalsFalse.asBooleanNotNull()); } @Test void getExpectedValueFloat64EqualsFloat64() { - ClickHouseConstant oneConst = ClickHouseConstant.createFloat64Constant(1); - ClickHouseConstant oneFConst = ClickHouseConstant.createFloat64Constant(1.0); - ClickHouseConstant zeroConst = ClickHouseConstant.createFloat64Constant(0); - ClickHouseConstant zeroFConst = ClickHouseConstant.createFloat64Constant(0.0); - ClickHouseConstant negativeConst = ClickHouseConstant.createFloat64Constant(-100.0); - ClickHouseConstant positiveConst = ClickHouseConstant.createFloat64Constant(10000.0); - - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneFConst.applyEquals(oneFConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneFConst).asInt(), 1); - - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneFConst.applyEquals(zeroFConst).asInt(), 0); - assertEquals(zeroConst.applyEquals(zeroFConst).asInt(), 1); - assertEquals(zeroFConst.applyEquals(zeroConst).asInt(), 1); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createFloat64Constant(1); + ClickHouseConstant oneFConst = ClickHouseCreateConstant.createFloat64Constant(1.0); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createFloat64Constant(0); + ClickHouseConstant zeroFConst = ClickHouseCreateConstant.createFloat64Constant(0.0); + + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneFConst.applyEquals(oneFConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneFConst).asBooleanNotNull()); + + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneFConst.applyEquals(zeroFConst).asBooleanNotNull()); + assertEquals(true, zeroConst.applyEquals(zeroFConst).asBooleanNotNull()); + assertEquals(true, zeroFConst.applyEquals(zeroConst).asBooleanNotNull()); } @Test void getExpectedValueInt32EqualsBool() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); - ClickHouseConstant oneConst = ClickHouseConstant.createInt32Constant(1); - ClickHouseConstant zeroConst = ClickHouseConstant.createInt32Constant(0); - ClickHouseConstant negativeConst = ClickHouseConstant.createInt32Constant(-100); - ClickHouseConstant positiveConst = ClickHouseConstant.createInt32Constant(10000); - - assertEquals(trueConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(falseConst.applyEquals(oneConst).asInt(), 0); - - assertEquals(trueConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(falseConst.applyEquals(zeroConst).asInt(), 1); - - assertEquals(negativeConst.applyEquals(oneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(trueConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(falseConst).asInt(), 0); - - assertEquals(positiveConst.applyEquals(oneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(trueConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(falseConst).asInt(), 0); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createInt32Constant(1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createInt32Constant(0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createInt32Constant(10000); + + assertEquals(true, trueConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, falseConst.applyEquals(oneConst).asBooleanNotNull()); + + assertEquals(false, trueConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, falseConst.applyEquals(zeroConst).asBooleanNotNull()); + + assertEquals(false, negativeConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(falseConst).asBooleanNotNull()); + + assertEquals(false, positiveConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(falseConst).asBooleanNotNull()); } @Test void getExpectedValueIntEqualsInt() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); for (ClickHouseDataType type : Arrays. stream(ClickHouseDataType.values()) .filter((dt) -> dt.name().contains("Int") && !dt.name().contains("Interval")) .collect(Collectors.toList())) { - ClickHouseConstant oneConst = ClickHouseConstant.createIntConstant(type, 1); - ClickHouseConstant zeroConst = ClickHouseConstant.createIntConstant(type, 0); - ClickHouseConstant negativeConst = ClickHouseConstant.createIntConstant(type, -100); - ClickHouseConstant positiveConst = ClickHouseConstant.createIntConstant(type, 10000); - - assertEquals(trueConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(falseConst.applyEquals(oneConst).asInt(), 0); - - assertEquals(trueConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(falseConst.applyEquals(zeroConst).asInt(), 1); - - assertEquals(negativeConst.applyEquals(oneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(trueConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(falseConst).asInt(), 0); - - assertEquals(positiveConst.applyEquals(oneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(trueConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(falseConst).asInt(), 0); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createIntConstant(type, 1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createIntConstant(type, 0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createIntConstant(type, -100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createIntConstant(type, 10000); + + assertEquals(true, trueConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, falseConst.applyEquals(oneConst).asBooleanNotNull()); + + assertEquals(false, trueConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, falseConst.applyEquals(zeroConst).asBooleanNotNull()); + + assertEquals(false, negativeConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(falseConst).asBooleanNotNull()); + + assertEquals(false, positiveConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(falseConst).asBooleanNotNull()); } } @Test void getExpectedValueInt32EqualsFloat64() { - ClickHouseConstant float64OneConst = ClickHouseConstant.createFloat64Constant(1.0); - ClickHouseConstant float64ZeroConst = ClickHouseConstant.createFloat64Constant(0.0); - ClickHouseConstant oneConst = ClickHouseConstant.createInt32Constant(1); - ClickHouseConstant zeroConst = ClickHouseConstant.createInt32Constant(0); - ClickHouseConstant negativeConst = ClickHouseConstant.createInt32Constant(-100); - ClickHouseConstant positiveConst = ClickHouseConstant.createInt32Constant(10000); + ClickHouseConstant float64OneConst = ClickHouseCreateConstant.createFloat64Constant(1.0); + ClickHouseConstant float64ZeroConst = ClickHouseCreateConstant.createFloat64Constant(0.0); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createInt32Constant(1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createInt32Constant(0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createInt32Constant(10000); - assertEquals(float64OneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(float64ZeroConst.applyEquals(oneConst).asInt(), 0); + assertEquals(true, float64OneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, float64ZeroConst.applyEquals(oneConst).asBooleanNotNull()); - assertEquals(float64OneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(float64ZeroConst.applyEquals(zeroConst).asInt(), 1); + assertEquals(false, float64OneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, float64ZeroConst.applyEquals(zeroConst).asBooleanNotNull()); - assertEquals(negativeConst.applyEquals(float64OneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(float64ZeroConst).asInt(), 0); + assertEquals(false, negativeConst.applyEquals(float64OneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(float64ZeroConst).asBooleanNotNull()); - assertEquals(positiveConst.applyEquals(float64OneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(float64ZeroConst).asInt(), 0); + assertEquals(false, positiveConst.applyEquals(float64OneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(float64ZeroConst).asBooleanNotNull()); } -} \ No newline at end of file +} diff --git a/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java b/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java new file mode 100644 index 000000000..fc786a625 --- /dev/null +++ b/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java @@ -0,0 +1,117 @@ +package sqlancer.clickhouse.ast; + +import org.junit.jupiter.api.Test; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ClickHouseOperatorsVisitorTest { + + @Test + void selectUnaryNot() { + ClickHouseConstant trueConst = ClickHouseCreateConstant.createBoolean(true); + ClickHouseExpression notTrue = new ClickHouseUnaryPrefixOperation(trueConst, + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.NOT); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(notTrue)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (NOT (true))"; + assertEquals(answer, result); + } + + @Test + void selectUnaryMinus() { + ClickHouseConstant fiveConst = ClickHouseCreateConstant.createUInt32Constant(5); + ClickHouseExpression minusFive = new ClickHouseUnaryPrefixOperation(fiveConst, + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.MINUS); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(minusFive)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (- (5))"; + assertEquals(answer, result); + } + + @Test + void selectUnaryExp() { + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression minusFive = new ClickHouseUnaryFunctionOperation(tenConst, + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.EXP); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(minusFive)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (exp (10))"; + assertEquals(answer, result); + } + + @Test + void selectBinaryPlus() { + ClickHouseConstant dConst = ClickHouseCreateConstant.createFloat32Constant((float) -1.1); + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression expr = new ClickHouseBinaryArithmeticOperation(dConst, tenConst, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.ADD); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT ((-1.1)+(10))"; + assertEquals(answer, result); + } + + @Test + void selectBinaryPow() { + ClickHouseConstant threeConst = ClickHouseCreateConstant.createInt8Constant(3); + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(threeConst, tenConst, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.POW); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT pow(3,10)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryLCM() { + ClickHouseConstant aConst = ClickHouseCreateConstant.createInt8Constant(100); + ClickHouseConstant bConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(aConst, bConst, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.LCM); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT lcm(100,-100)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryDivCol() { + ClickHouseColumnReference a = new ClickHouseColumnReference(new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null), null, null); + ClickHouseColumnReference b = new ClickHouseColumnReference(new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null), null, null); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(a, b, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.INT_DIV); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT intDiv(a,b)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryComp() { + ClickHouseConstant aConst = ClickHouseCreateConstant.createInt8Constant(10); + ClickHouseConstant bConst = ClickHouseCreateConstant.createInt32Constant(100); + ClickHouseExpression expr = new ClickHouseBinaryComparisonOperation(aConst, bConst, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT ((10)>(100))"; + assertEquals(answer, result); + } + +} diff --git a/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java new file mode 100644 index 000000000..ba9628037 --- /dev/null +++ b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java @@ -0,0 +1,344 @@ +package sqlancer.clickhouse.ast; + +import org.junit.jupiter.api.Test; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.constant.ClickHouseInt8Constant; +import sqlancer.common.schema.TableIndex; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ClickHouseToStringVisitorTest { + + @Test + void select1Test() { + ClickHouseConstant oneConst = new ClickHouseInt8Constant(1); + ClickHouseSelect selectOne = new ClickHouseSelect(); + selectOne.setFetchColumns(Arrays.asList(oneConst)); + String result = ClickHouseVisitor.asString(selectOne); + String answer = "SELECT 1"; + assertEquals(answer, result); + } + + @Test + void select1asATest() { + ClickHouseAliasOperation oneConstAsA = new ClickHouseAliasOperation(new ClickHouseInt8Constant(1), "a"); + ClickHouseSelect selectOne = new ClickHouseSelect(); + selectOne.setFetchColumns(Arrays.asList(oneConstAsA)); + String result = ClickHouseVisitor.asString(selectOne); + String answer = "SELECT 1 AS `a`"; + assertEquals(answer, result); + } + + @Test + void selectATest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseSelect selectA = new ClickHouseSelect(); + selectA.setFetchColumns(Arrays.asList(a_ref)); + selectA.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectA); + String answer = "SELECT t.a FROM t"; + assertEquals(answer, result); + } + + @Test + void selectAasBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseAliasOperation b = new ClickHouseAliasOperation(a_ref, "b"); + ClickHouseColumnReference b_ref = new ClickHouseColumnReference(b); + ClickHouseSelect selectA = new ClickHouseSelect(); + selectA.setFetchColumns(Arrays.asList(b, b_ref)); + selectA.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectA); + String answer = "SELECT t.a AS `b`, b FROM t"; + assertEquals(answer, result); + } + + @Test + void selectABTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t"; + assertEquals(answer, result); + } + + @Test + void selectWhereAGreaterBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + selectAB.setWhereClause(new ClickHouseBinaryComparisonOperation(a_ref, b_ref, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t WHERE ((t.a)>(t.b))"; + assertEquals(answer, result); + } + + @Test + void selectWhereAGreaterConstTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + ClickHouseConstant c_const = new ClickHouseInt8Constant(1); + selectAB.setWhereClause(new ClickHouseBinaryComparisonOperation(a_ref, c_const, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t WHERE ((t.a)>(1))"; + assertEquals(answer, result); + } + + @Test + void selectSumAGroupByBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + ClickHouseAggregate sum_a = new ClickHouseAggregate(a_ref, ClickHouseAggregate.ClickHouseAggregateFunction.SUM); + selectAB.setFetchColumns(Arrays.asList(sum_a)); + selectAB.setFromClause(table_ref); + selectAB.setGroupByClause(Arrays.asList(b_ref)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT SUM(t.a) FROM t GROUP BY t.b"; + assertEquals(answer, result); + } + + @Test + void selectCrossJoinTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", empty_col_list, indexes, + false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", empty_col_list, indexes, + false); + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, null); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, null); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table1); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table1); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table2); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table2); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseColumnReference a1_ref = a1_col.asColumnReference(null); + ClickHouseColumnReference b1_ref = b1_col.asColumnReference(null); + ClickHouseColumnReference a2_ref = a2_col.asColumnReference(null); + ClickHouseColumnReference b2_ref = b2_col.asColumnReference(null); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.CROSS); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT t1.a1, t2.a2, t1.b1, t2.b2 FROM t1 JOIN t2"; + assertEquals(answer, result); + } + + @Test + void selectCrossJoinAliasedTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, "left"); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, "right"); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference a2_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference b1_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.CROSS); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT left.a1, left.b1, right.a2, right.b2 FROM t1 AS left JOIN t2 AS right"; + assertEquals(answer, result); + } + + @Test + void selectJoinONTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, null); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, null); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference b1_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference a2_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoinOnClause on = new ClickHouseExpression.ClickHouseJoinOnClause(a1_ref, + a2_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.INNER, on); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT t1.a1, t2.a2, t1.b1, t2.b2 FROM t1 INNER JOIN t2 ON ((t1.a1)=(t2.a2))"; + assertEquals(answer, result); + } + + @Test + void selectJoinONAliasedTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, "left"); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, "right"); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference b1_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference a2_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoinOnClause on = new ClickHouseExpression.ClickHouseJoinOnClause(a1_ref, + a2_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.INNER, on); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT left.a1, right.a2, left.b1, right.b2 FROM t1 AS left INNER JOIN t2 AS right ON ((left.a1)=(right.a2))"; + assertEquals(answer, result); + } +} diff --git a/test/sqlancer/common/query/SQLQueryErrorTest.java b/test/sqlancer/common/query/SQLQueryErrorTest.java new file mode 100644 index 000000000..dea1e7a53 --- /dev/null +++ b/test/sqlancer/common/query/SQLQueryErrorTest.java @@ -0,0 +1,92 @@ +package sqlancer.common.query; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SQLQueryErrorTest { + @Test + public void testSettersAndGetters() { + SQLQueryError error = new SQLQueryError(); + error.setLevel(SQLQueryError.ErrorLevel.ERROR); + error.setCode(123); + error.setMessage("Test message"); + assertEquals(SQLQueryError.ErrorLevel.ERROR, error.getLevel()); + assertEquals(123, error.getCode()); + assertEquals("Test message", error.getMessage()); + } + + @Test + public void testHasSameLevel() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setLevel(SQLQueryError.ErrorLevel.WARNING); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); + assertTrue(e1.hasSameLevel(e2)); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); + assertFalse(e1.hasSameLevel(e2)); + } + + @Test + public void testHasSameCodeAndMessage() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setCode(1); + e2.setCode(1); + e1.setMessage("msg"); + e2.setMessage("msg"); + assertTrue(e1.hasSameCodeAndMessage(e2)); + e2.setCode(2); + assertFalse(e1.hasSameCodeAndMessage(e2)); + e2.setCode(1); + e2.setMessage("other"); + assertFalse(e1.hasSameCodeAndMessage(e2)); + } + + @Test + public void testEquals() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setLevel(SQLQueryError.ErrorLevel.ERROR); + e1.setCode(1); + e1.setMessage("msg"); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); + e2.setCode(1); + e2.setMessage("msg"); + assertEquals(e1, e2); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); + assertNotEquals(e1, e2); + } + + @Test + public void testToString() { + SQLQueryError e = new SQLQueryError(); + e.setLevel(SQLQueryError.ErrorLevel.ERROR); + e.setCode(1); + e.setMessage("msg"); + String str = e.toString(); + assertTrue(str.contains("Level: ERROR")); + assertTrue(str.contains("Code: 1")); + assertTrue(str.contains("Message: msg")); + } + + @Test + public void testCompareTo() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setCode(1); + e2.setCode(2); + assertTrue(e1.compareTo(e2) < 0); + e2.setCode(1); + e1.setLevel(SQLQueryError.ErrorLevel.ERROR); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); + assertTrue(e1.compareTo(e2) > 0 || e1.compareTo(e2) < 0); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); + e1.setMessage("a"); + e2.setMessage("b"); + assertTrue(e1.compareTo(e2) < 0); + } +} diff --git a/test/sqlancer/dbms/TestCitus.java b/test/sqlancer/dbms/TestCitus.java new file mode 100644 index 000000000..3ef4a7e50 --- /dev/null +++ b/test/sqlancer/dbms/TestCitus.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCitus { + + @Test + public void testCitus() { + String citusAvailable = System.getenv("CITUS_AVAILABLE"); + boolean citusIsAvailable = citusAvailable != null && citusAvailable.equalsIgnoreCase("true"); + assumeTrue(citusIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "citus", "--connection-url", + "postgresql://localhost:9700/test", "--test-collations", "false" })); + } + +} diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 5626ac3b3..87d7713fe 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -3,49 +3,136 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; + import sqlancer.Main; -import sqlancer.dbms.TestConfig; public class TestClickHouse { @Test - public void testClickHouseTLPWhereHaving() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); - Assertions.assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", TestConfig.SECONDS, "--num-queries", - TestConfig.NUM_QUERIES, "--num-threads", "50", "clickhouse", "--oracle", "TLPWhere", "--oracle", - "TLPHaving" })); + public void testClickHouseTLPWhereGroupBy() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T1_", "clickhouse", + "--oracle", "TLPWhere", "--oracle", "TLPGroupBy")); } @Test public void testClickHouseTLPWhere() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); - Assertions.assertEquals(0, - Main.executeMain(new String[] { "--timeout-seconds", TestConfig.SECONDS, "--num-queries", - TestConfig.NUM_QUERIES, "--num-threads", "50", "clickhouse", "--oracle", "TLPWhere" })); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T2_", "clickhouse", + "--oracle", "TLPWhere")); } @Test public void testClickHouseTLPHaving() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); - assertEquals(0, Main.executeMain(new String[] { "--timeout-seconds", TestConfig.SECONDS, "--num-queries", - TestConfig.NUM_QUERIES, "--num-threads", "30", "clickhouse", "--oracle", "TLPHaving" })); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "--database-prefix", "T3_", "clickhouse", "--oracle", "TLPHaving")); } @Test public void testClickHouseTLPGroupBy() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); - assertEquals(0, Main.executeMain(new String[] { "--timeout-seconds", TestConfig.SECONDS, "--num-queries", - TestConfig.NUM_QUERIES, "--num-threads", "30", "clickhouse", "--oracle", "TLPGroupBy" })); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T4_", "clickhouse", "--oracle", "TLPGroupBy")); + } + + @Test + public void testClickHouseTLPDistinct() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T5_", "clickhouse", "--oracle", "TLPDistinct")); + } + + @Test + public void testClickHouseTLPAggregate() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T6_", "clickhouse", "--oracle", "TLPAggregate")); + } + + @Test + public void testClickHouseNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "--database-prefix", "T7_", "clickhouse", "--oracle", "NoREC")); + } + + @Test + public void testClickHouseTLPWhereGroupByWithJoins() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T8_", "clickhouse", + "--oracle", "TLPWhere", "--oracle", "TLPGroupBy")); + } + + @Test + public void testClickHouseTLPWhereWithJoins() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T9_", "clickhouse", + "--oracle", "TLPWhere")); } + + @Test + public void testClickHouseTLPHavingWithJoins() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "--database-prefix", "T10_", "clickhouse", "--oracle", "TLPHaving")); + } + + @Test + public void testClickHouseTLPGroupByWithJoins() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T11_", "clickhouse", "--oracle", "TLPGroupBy")); + } + + @Test + public void testClickHouseTLPDistinctWithJoins() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T12_", "clickhouse", "--oracle", "TLPDistinct")); + } + + @Test + public void testClickHouseTLPAggregateWithJoins() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T13_", "clickhouse", "--oracle", "TLPAggregate")); + } + + @Test + public void testClickHouseNoRECWithJoins() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "--database-prefix", "T14_", "clickhouse", "--oracle", "NoREC")); + } + } diff --git a/test/sqlancer/dbms/TestCockroachDBCERT.java b/test/sqlancer/dbms/TestCockroachDBCERT.java new file mode 100644 index 000000000..4d7140e40 --- /dev/null +++ b/test/sqlancer/dbms/TestCockroachDBCERT.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCockroachDBCERT { + + @Test + public void testCockroachDBCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "CERT" })); + } + +} diff --git a/test/sqlancer/dbms/TestCockroachDB.java b/test/sqlancer/dbms/TestCockroachDBNoREC.java similarity index 58% rename from test/sqlancer/dbms/TestCockroachDB.java rename to test/sqlancer/dbms/TestCockroachDBNoREC.java index 5eb701e3b..bf1a51193 100644 --- a/test/sqlancer/dbms/TestCockroachDB.java +++ b/test/sqlancer/dbms/TestCockroachDBNoREC.java @@ -7,15 +7,13 @@ import sqlancer.Main; -public class TestCockroachDB { +public class TestCockroachDBNoREC { @Test - public void testMySQL() { - String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); - boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); - assumeTrue(cockroachDBIsAvailable); + public void testCockroachDBNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb" })); + "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "NOREC" })); } } diff --git a/test/sqlancer/dbms/TestCockroachDBTLP.java b/test/sqlancer/dbms/TestCockroachDBTLP.java new file mode 100644 index 000000000..f916844ed --- /dev/null +++ b/test/sqlancer/dbms/TestCockroachDBTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCockroachDBTLP { + + @Test + public void testCockroachDBTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 9fac06920..f2372f266 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -3,4 +3,25 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + + public static final String CLICKHOUSE_ENV = "CLICKHOUSE_AVAILABLE"; + public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; + public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; + public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; + public static final String DORIS_ENV = "DORIS_AVAILABLE"; + public static final String HIVE_ENV = "HIVE_AVAILABLE"; + public static final String SPARK_ENV = "SPARK_AVAILABLE"; + public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; + public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; + public static final String MYSQL_ENV = "MYSQL_AVAILABLE"; + public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; + public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; + public static final String PRESTO_ENV = "PRESTO_AVAILABLE"; + public static final String TIDB_ENV = "TIDB_AVAILABLE"; + public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; + + public static boolean isEnvironmentTrue(String key) { + String value = System.getenv(key); + return value != null && value.equalsIgnoreCase("true"); + } } diff --git a/test/sqlancer/dbms/TestDataFusion.java b/test/sqlancer/dbms/TestDataFusion.java new file mode 100644 index 000000000..b2b5e2a1a --- /dev/null +++ b/test/sqlancer/dbms/TestDataFusion.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestDataFusion { + @Test + public void testDataFusion() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATAFUSION_ENV)); + + assertEquals(0, Main.executeMain("--random-seed", "0", "--num-threads", "1", // TODO(datafusion) update when + // multithread is supported + "--timeout-seconds", TestConfig.SECONDS, "--num-queries", TestConfig.NUM_QUERIES, "datafusion")); + } +} diff --git a/test/sqlancer/dbms/TestDatabendNoREC.java b/test/sqlancer/dbms/TestDatabendNoREC.java new file mode 100644 index 000000000..679f8c161 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendNoREC.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.Randomly; + +public class TestDatabendNoREC { + + @Test + public void testDatabendNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); + } + +} diff --git a/test/sqlancer/dbms/TestDatabendPQS.java b/test/sqlancer/dbms/TestDatabendPQS.java new file mode 100644 index 000000000..fba733d44 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendPQS.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.Randomly; + +public class TestDatabendPQS { + + @Test + public void testDatabendPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); + } + +} diff --git a/test/sqlancer/dbms/TestDatabendTLP.java b/test/sqlancer/dbms/TestDatabendTLP.java new file mode 100644 index 000000000..27ba53416 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendTLP.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDatabendTLP { + + @Test + public void testDatabendTLPQueryPartitioning() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); + } +} diff --git a/test/sqlancer/dbms/TestDorisNoREC.java b/test/sqlancer/dbms/TestDorisNoREC.java new file mode 100644 index 000000000..76a9d50a8 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisNoREC.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisNoREC { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "NOREC")); + } +} diff --git a/test/sqlancer/dbms/TestDorisPQS.java b/test/sqlancer/dbms/TestDorisPQS.java new file mode 100644 index 000000000..5760003d3 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisPQS.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisPQS { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "PQS")); + } +} diff --git a/test/sqlancer/dbms/TestDorisTLP.java b/test/sqlancer/dbms/TestDorisTLP.java new file mode 100644 index 000000000..f8bea9e47 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisTLP.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisTLP { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "QUERY_PARTITIONING")); + } +} diff --git a/test/sqlancer/dbms/TestSQLite3.java b/test/sqlancer/dbms/TestDuckDBNoREC.java similarity index 68% rename from test/sqlancer/dbms/TestSQLite3.java rename to test/sqlancer/dbms/TestDuckDBNoREC.java index f26b449c1..dea1c70dd 100644 --- a/test/sqlancer/dbms/TestSQLite3.java +++ b/test/sqlancer/dbms/TestDuckDBNoREC.java @@ -6,13 +6,11 @@ import sqlancer.Main; -public class TestSQLite3 { - +public class TestDuckDBNoREC { @Test - public void testSqlite() { + public void testDuckDBNoREC() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", "0", "sqlite3" })); + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", "NOREC" })); } - } diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDBTLP.java similarity index 66% rename from test/sqlancer/dbms/TestDuckDB.java rename to test/sqlancer/dbms/TestDuckDBTLP.java index ad62befbf..322e3eeb4 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDBTLP.java @@ -6,13 +6,11 @@ import sqlancer.Main; -public class TestDuckDB { +public class TestDuckDBTLP { @Test - public void testDuckDB() { + public void testDuckDBTLP() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 - assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", "NoREC" })); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", diff --git a/test/sqlancer/dbms/TestH2.java b/test/sqlancer/dbms/TestH2.java new file mode 100644 index 000000000..ec6962101 --- /dev/null +++ b/test/sqlancer/dbms/TestH2.java @@ -0,0 +1,18 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestH2 { + + @Test + public void testH2DB() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "h2" })); + + } + +} diff --git a/test/sqlancer/dbms/TestHSQLDBNoREC.java b/test/sqlancer/dbms/TestHSQLDBNoREC.java new file mode 100644 index 000000000..721fbb126 --- /dev/null +++ b/test/sqlancer/dbms/TestHSQLDBNoREC.java @@ -0,0 +1,15 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHSQLDBNoREC { + @Test + public void testHSQLDBNoREC() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "hsqldb", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestHSQLDBTLP.java b/test/sqlancer/dbms/TestHSQLDBTLP.java new file mode 100644 index 000000000..a0b9c18d9 --- /dev/null +++ b/test/sqlancer/dbms/TestHSQLDBTLP.java @@ -0,0 +1,15 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHSQLDBTLP { + @Test + public void testHSQLDBTLP() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "hsqldb", "--oracle", "WHERE" })); + } +} diff --git a/test/sqlancer/dbms/TestHiveTLP.java b/test/sqlancer/dbms/TestHiveTLP.java new file mode 100644 index 000000000..5f8692f3f --- /dev/null +++ b/test/sqlancer/dbms/TestHiveTLP.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHiveTLP { + + @Test + public void testHiveTLPWhere() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.HIVE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "hive", "--oracle", "TLPWhere" })); + } +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestMariaDB.java b/test/sqlancer/dbms/TestMariaDB.java index e5c188d28..eb26bfb4a 100644 --- a/test/sqlancer/dbms/TestMariaDB.java +++ b/test/sqlancer/dbms/TestMariaDB.java @@ -3,19 +3,45 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import java.util.List; +import java.util.stream.Collectors; + import org.junit.jupiter.api.Test; import sqlancer.Main; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.ast.MariaDBColumnName; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBTableReference; +import sqlancer.mariadb.ast.MariaDBVisitor; public class TestMariaDB { + @Test + public void testSelectAsString() { + MariaDBSchema.MariaDBColumn c0 = new MariaDBSchema.MariaDBColumn("c0", MariaDBSchema.MariaDBDataType.INT, true, + 0); + MariaDBSchema.MariaDBColumn c1 = new MariaDBSchema.MariaDBColumn("c1", MariaDBSchema.MariaDBDataType.INT, false, + 0); + List columns = List.of(c0, c1); + List indices = List.of(); + MariaDBSchema.MariaDBTable t1 = new MariaDBSchema.MariaDBTable("t1", columns, indices, + MariaDBSchema.MariaDBTable.MariaDBEngine.INNO_DB); + MariaDBSchema.MariaDBTables tables = new MariaDBSchema.MariaDBTables(List.of(t1)); + + MariaDBSelectStatement select = new MariaDBSelectStatement(); + select.setFetchColumns(tables.getColumns().stream().map(MariaDBColumnName::new).collect(Collectors.toList())); + select.setFromList(tables.getTables().stream().map(MariaDBTableReference::new).collect(Collectors.toList())); + + String selectString = MariaDBVisitor.asString(select); + assertEquals("SELECT c0, c1 FROM t1", selectString); + } + @Test public void testMariaDB() { - String mariaDBAvailable = System.getenv("MARIADB_AVAILABLE"); - boolean mariaDBIsAvailable = mariaDBAvailable != null && mariaDBAvailable.equalsIgnoreCase("true"); - assumeTrue(mariaDBIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MARIADB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", "0", "mariadb" })); + "--num-queries", TestConfig.NUM_QUERIES, "mariadb" })); } } diff --git a/test/sqlancer/dbms/TestMaterializeNoREC.java b/test/sqlancer/dbms/TestMaterializeNoREC.java new file mode 100644 index 000000000..e4ac980ce --- /dev/null +++ b/test/sqlancer/dbms/TestMaterializeNoREC.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterializeNoREC { + + @Test + public void test() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--username", "materialize", + "materialize", "--oracle", "NOREC", "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/dbms/TestMaterializePQS.java b/test/sqlancer/dbms/TestMaterializePQS.java new file mode 100644 index 000000000..0c6be974d --- /dev/null +++ b/test/sqlancer/dbms/TestMaterializePQS.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterializePQS { + + @Test + public void test() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "--username", "materialize", "materialize", "--oracle", "pqs", + "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/dbms/TestMaterializeTLP.java b/test/sqlancer/dbms/TestMaterializeTLP.java new file mode 100644 index 000000000..a98b3e053 --- /dev/null +++ b/test/sqlancer/dbms/TestMaterializeTLP.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterializeTLP { + + @Test + public void test() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--username", "materialize", + "materialize", "--set-max-tables-mvs", "true", "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/dbms/TestMySQL.java b/test/sqlancer/dbms/TestMySQLCERT.java similarity index 58% rename from test/sqlancer/dbms/TestMySQL.java rename to test/sqlancer/dbms/TestMySQLCERT.java index 9d7c2d8e3..a7704a545 100644 --- a/test/sqlancer/dbms/TestMySQL.java +++ b/test/sqlancer/dbms/TestMySQLCERT.java @@ -7,17 +7,16 @@ import sqlancer.Main; -public class TestMySQL { +public class TestMySQLCERT { @Test public void testMySQL() { - String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); - boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); - assumeTrue(mysqlIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); + // Run with 0 queries as there are false positives for every mutation assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--max-expression-depth", "1", "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, - "mysql" })); + "--max-expression-depth", "1", "--num-threads", "1", "--num-queries", "0", "mysql", "--oracle", + "CERT" })); } } diff --git a/test/sqlancer/dbms/TestMySQLDQE.java b/test/sqlancer/dbms/TestMySQLDQE.java new file mode 100644 index 000000000..c137c1d0a --- /dev/null +++ b/test/sqlancer/dbms/TestMySQLDQE.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestMySQLDQE { + + @Test + public void testMySQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); + // Run with 0 queries as there are false positives for every mutation + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, + "mysql", "--oracle", "DQE" })); + } + +} diff --git a/test/sqlancer/dbms/TestMySQLPQS.java b/test/sqlancer/dbms/TestMySQLPQS.java new file mode 100644 index 000000000..6f1b0786f --- /dev/null +++ b/test/sqlancer/dbms/TestMySQLPQS.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMySQLPQS { + + @Test + public void testPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC", "--database-prefix", + "pqsdb" /* Workaround for connections not being closed */, "--num-queries", + TestConfig.NUM_QUERIES, "mysql", "--oracle", "PQS" })); + } + +} diff --git a/test/sqlancer/dbms/TestMySQLTLP.java b/test/sqlancer/dbms/TestMySQLTLP.java new file mode 100644 index 000000000..6254c58f3 --- /dev/null +++ b/test/sqlancer/dbms/TestMySQLTLP.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMySQLTLP { + + @Test + public void testMySQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, + "mysql", "--oracle", "TLP_WHERE" })); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java new file mode 100644 index 000000000..c687590de --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBaseNoREC { + + @Test + public void testNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--database-prefix", "norecdb", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "NoREC" })); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java new file mode 100644 index 000000000..404977398 --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -0,0 +1,27 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBasePQS { + + @Test + public void testPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", + "--database-prefix", "pqsdb", "--num-queries", TestConfig.NUM_QUERIES, "--username", + "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "PQS" })); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java new file mode 100644 index 000000000..cf86c2078 --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBaseTLP { + + @Test + public void testTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--database-prefix", "tlpdb", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "TLP_WHERE" })); + } + +} diff --git a/test/sqlancer/dbms/TestPostgresCERT.java b/test/sqlancer/dbms/TestPostgresCERT.java new file mode 100644 index 000000000..afa5e3801 --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresCERT.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresCERT { + + @Test + public void testCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", + "false", "--oracle", "CERT" })); + } +} diff --git a/test/sqlancer/dbms/TestPostgresNoREC.java b/test/sqlancer/dbms/TestPostgresNoREC.java new file mode 100644 index 000000000..8b9f00d48 --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresNoREC.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresNoREC { + + @Test + public void testNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", + "false", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestPostgresPQS.java b/test/sqlancer/dbms/TestPostgresPQS.java new file mode 100644 index 000000000..f6a37f533 --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresPQS.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresPQS { + + @Test + public void testPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "postgres", "--test-collations", "false", "--oracle", "pqs" })); + } + +} diff --git a/test/sqlancer/dbms/TestPostgres.java b/test/sqlancer/dbms/TestPostgresTLP.java similarity index 64% rename from test/sqlancer/dbms/TestPostgres.java rename to test/sqlancer/dbms/TestPostgresTLP.java index 3bb64ef80..5bd722991 100644 --- a/test/sqlancer/dbms/TestPostgres.java +++ b/test/sqlancer/dbms/TestPostgresTLP.java @@ -7,17 +7,14 @@ import sqlancer.Main; -public class TestPostgres { +public class TestPostgresTLP { @Test - public void testPostgres() { - String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); - boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); - assumeTrue(postgresIsAvailable); + public void testTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", "false" })); } - } diff --git a/test/sqlancer/dbms/TestPrestoNoREC.java b/test/sqlancer/dbms/TestPrestoNoREC.java new file mode 100644 index 000000000..e38e90483 --- /dev/null +++ b/test/sqlancer/dbms/TestPrestoNoREC.java @@ -0,0 +1,17 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPrestoNoREC { + @Test + public void testPrestoNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.PRESTO_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "presto", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestPrestoTLP.java b/test/sqlancer/dbms/TestPrestoTLP.java new file mode 100644 index 000000000..6ffefc333 --- /dev/null +++ b/test/sqlancer/dbms/TestPrestoTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPrestoTLP { + @Test + public void testPrestoTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.PRESTO_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--validate-result-size-only", + "true", "--canonicalize-sql-strings", "false", "presto", "--oracle", "QUERY_PARTITIONING" })); + } +} diff --git a/test/sqlancer/dbms/TestSQLiteCODDTest.java b/test/sqlancer/dbms/TestSQLiteCODDTest.java new file mode 100644 index 000000000..c1948a9b8 --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteCODDTest.java @@ -0,0 +1,16 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteCODDTest { + + @Test + public void testSqliteCODDTest() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", "CODDTest" })); + } +} diff --git a/test/sqlancer/dbms/TestSQLiteNoREC.java b/test/sqlancer/dbms/TestSQLiteNoREC.java new file mode 100644 index 000000000..aa7741659 --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteNoREC.java @@ -0,0 +1,17 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteNoREC { + + @Test + public void testSqliteNoREC() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", "NoREC" })); + } + +} diff --git a/test/sqlancer/dbms/TestSQLitePQS.java b/test/sqlancer/dbms/TestSQLitePQS.java new file mode 100644 index 000000000..976b305b6 --- /dev/null +++ b/test/sqlancer/dbms/TestSQLitePQS.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLitePQS { + + @Test + public void testSqlitePQS() { + // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", "1000", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", + "--database-prefix", "pqsdb" /* Workaround for connections not being closed */, "sqlite3", "--oracle", + "PQS", "--test-fts", "false", "--test-rtree", "false", "--test-check-constraints", "false", + "--test-in-operator", "false" })); + } + +} diff --git a/test/sqlancer/dbms/TestSQLiteTLP.java b/test/sqlancer/dbms/TestSQLiteTLP.java new file mode 100644 index 000000000..7d90fd93c --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteTLP { + + @Test + public void testSqliteTLP() { + // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", + "QUERY_PARTITIONING" })); + } +} diff --git a/test/sqlancer/dbms/TestSparkTLP.java b/test/sqlancer/dbms/TestSparkTLP.java new file mode 100644 index 000000000..83302ceff --- /dev/null +++ b/test/sqlancer/dbms/TestSparkTLP.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSparkTLP { + + @Test + public void testSparkTLPWhere() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.SPARK_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "spark", "--oracle", "TLPWhere" })); + } +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestTiDBCERT.java b/test/sqlancer/dbms/TestTiDBCERT.java new file mode 100644 index 000000000..748444870 --- /dev/null +++ b/test/sqlancer/dbms/TestTiDBCERT.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestTiDBCERT { + + @Test + public void testCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.TIDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", "4", "tidb", "--oracle", "CERT" })); + } + +} diff --git a/test/sqlancer/dbms/TestTiDBTLP.java b/test/sqlancer/dbms/TestTiDBTLP.java new file mode 100644 index 000000000..11cf7581e --- /dev/null +++ b/test/sqlancer/dbms/TestTiDBTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestTiDBTLP { + + @Test + public void testTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.TIDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", "0", "tidb" })); + } + +} diff --git a/test/sqlancer/dbms/TestYCQL.java b/test/sqlancer/dbms/TestYCQL.java new file mode 100644 index 000000000..1198e2376 --- /dev/null +++ b/test/sqlancer/dbms/TestYCQL.java @@ -0,0 +1,18 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestYCQL { + @Test + public void testYCQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "cassandra", "--password", "cassandra", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ycql")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLNoREC.java b/test/sqlancer/dbms/TestYSQLNoREC.java new file mode 100644 index 000000000..51f0487cc --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLNoREC.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLNoREC { + @Test + public void testYSQLNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "NOREC")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLPQS.java b/test/sqlancer/dbms/TestYSQLPQS.java new file mode 100644 index 000000000..6a21b6cb2 --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLPQS.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLPQS { + @Test + public void testYSQLPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "PQS")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLTLP.java b/test/sqlancer/dbms/TestYSQLTLP.java new file mode 100644 index 000000000..fdcdf372a --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLTLP { + @Test + public void testYSQLTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "QUERY_PARTITIONING")); + } +} diff --git a/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java b/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java new file mode 100644 index 000000000..47dabc225 --- /dev/null +++ b/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java @@ -0,0 +1,22 @@ +package sqlancer.dqp.mariadb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMariaDBDQP { + + @Test + public void testMariaDBDQPMethod() { + String mariadb = System.getenv("MARIADB_AVAILABLE"); + boolean mariadbIsAvailable = mariadb != null && mariadb.equalsIgnoreCase("true"); + assumeTrue(mariadbIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "mariadb", "--oracle", "DQP" })); + } + +} diff --git a/test/sqlancer/dqp/mysql/TestMySQLDQP.java b/test/sqlancer/dqp/mysql/TestMySQLDQP.java new file mode 100644 index 000000000..38e5eef61 --- /dev/null +++ b/test/sqlancer/dqp/mysql/TestMySQLDQP.java @@ -0,0 +1,22 @@ +package sqlancer.dqp.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMySQLDQP { + + @Test + public void testmysqlQPG() { + String mysql = System.getenv("MYSQL_AVAILABLE"); + boolean mysqlIsAvailable = mysql != null && mysql.equalsIgnoreCase("true"); + assumeTrue(mysqlIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "mysql", "--oracle", "DQP" })); + } + +} diff --git a/test/sqlancer/dbms/TestTiDB.java b/test/sqlancer/dqp/tidb/TestTiDBDQP.java similarity index 68% rename from test/sqlancer/dbms/TestTiDB.java rename to test/sqlancer/dqp/tidb/TestTiDBDQP.java index b97e59273..300dfde34 100644 --- a/test/sqlancer/dbms/TestTiDB.java +++ b/test/sqlancer/dqp/tidb/TestTiDBDQP.java @@ -1,4 +1,4 @@ -package sqlancer.dbms; +package sqlancer.dqp.tidb; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; @@ -6,16 +6,17 @@ import org.junit.jupiter.api.Test; import sqlancer.Main; +import sqlancer.dbms.TestConfig; -public class TestTiDB { +public class TestTiDBDQP { @Test - public void testMySQL() { + public void testTiDBQPG() { String tiDB = System.getenv("TIDB_AVAILABLE"); boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); assumeTrue(tiDBIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", "0", "tidb" })); + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "tidb", "--oracle", "DQP" })); } } diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java new file mode 100644 index 000000000..d39c2695c --- /dev/null +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -0,0 +1,77 @@ +package sqlancer.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; + +public class MySQLToStringVisitorTest { + + @Test + void visitAggregateToString() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); + + MySQLAggregate aggrCount = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT); + assertEquals("COUNT(a)", MySQLVisitor.asString(aggrCount)); + + MySQLAggregate aggrSum = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM); + assertEquals("SUM(a)", MySQLVisitor.asString(aggrSum)); + + MySQLAggregate aggrMin = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN); + assertEquals("MIN(a)", MySQLVisitor.asString(aggrMin)); + + MySQLAggregate aggrMax = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX); + assertEquals("MAX(a)", MySQLVisitor.asString(aggrMax)); + } + + @Test + void visitAggregateWithDistinctToString() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); + + MySQLAggregate aggrCountDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT); + assertEquals("COUNT(DISTINCT a)", MySQLVisitor.asString(aggrCountDistinct)); + + MySQLAggregate aggrSumDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.SUM_DISTINCT); + assertEquals("SUM(DISTINCT a)", MySQLVisitor.asString(aggrSumDistinct)); + + MySQLAggregate aggrMinDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.MIN_DISTINCT); + assertEquals("MIN(DISTINCT a)", MySQLVisitor.asString(aggrMinDistinct)); + + MySQLAggregate aggrMaxDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT); + assertEquals("MAX(DISTINCT a)", MySQLVisitor.asString(aggrMaxDistinct)); + } + + @Test + void visitCaseWhenToString() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals("(CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 0 END)", + MySQLVisitor.asString(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr))); + assertEquals("(CASE WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 0 END)", + MySQLVisitor.asString(new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr))); + assertEquals("(CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 END)", + MySQLVisitor.asString(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null))); + assertEquals("(CASE WHEN 1 THEN 11 WHEN 2 THEN 22 END)", + MySQLVisitor.asString(new MySQLCaseOperator(null, whenExprs, thenExprs, null))); + } +} diff --git a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java new file mode 100644 index 000000000..757241445 --- /dev/null +++ b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java @@ -0,0 +1,67 @@ +package sqlancer.mysql.ast; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; + +public class MySQLCaseOperatorTest { + + @Test + void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createIntConstant(1)); + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + MySQLCaseOperator caseOperator = new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr); + + assertEquals(11, caseOperator.getExpectedValue().getInt()); + } + + @Test + void getExpectedValue_switchConditionHasNoMatches_ReturnsElse() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createNullConstant()); + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals(0, new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr).getExpectedValue().getInt()); + assertTrue(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null).getExpectedValue().isNull()); + } + + @Test + void getExpectedValue_whenTrue_ReturnsThen() { + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + MySQLCaseOperator caseOperator = new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr); + + assertEquals(11, caseOperator.getExpectedValue().getInt()); + } + + @Test + void getExpectedValue_whenAllFalse_ReturnsElse() { + List whenExprs = List.of(MySQLIntConstant.createBoolean(false), + MySQLIntConstant.createBoolean(false)); + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals(0, new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr).getExpectedValue().getInt()); + assertTrue(new MySQLCaseOperator(null, whenExprs, thenExprs, null).getExpectedValue().isNull()); + } +} diff --git a/test/sqlancer/postgres/TestPostgresProvider.java b/test/sqlancer/postgres/TestPostgresProvider.java new file mode 100644 index 000000000..81e795568 --- /dev/null +++ b/test/sqlancer/postgres/TestPostgresProvider.java @@ -0,0 +1,21 @@ +package sqlancer.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class TestPostgresProvider { + + @Test + void createTablespaceIsNotScheduledWhenDisabled() { + PostgresGlobalState state = new PostgresGlobalState(); + state.setDbmsSpecificOptions(new PostgresOptions() { + @Override + public boolean isTestTablespaces() { + return false; + } + }); + + assertEquals(0, PostgresProvider.mapActions(state, PostgresProvider.Action.CREATE_TABLESPACE)); + } +} diff --git a/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java b/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java new file mode 100644 index 000000000..c2f626041 --- /dev/null +++ b/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.postgres.gen; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import sqlancer.IgnoreMeException; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; + +class TestPostgresTableSpaceGenerator { + + @Test + void generateIsSkippedWhenTablespacesAreDisabled() { + PostgresGlobalState state = new PostgresGlobalState(); + state.setDbmsSpecificOptions(new PostgresOptions() { + @Override + public boolean isTestTablespaces() { + return false; + } + }); + + assertThrows(IgnoreMeException.class, () -> PostgresTableSpaceGenerator.generate(state)); + } +} diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java b/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java new file mode 100644 index 000000000..b257a529b --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToBlob.java @@ -0,0 +1,44 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.ast.SQLite3Cast; +import sqlancer.sqlite3.ast.SQLite3Constant; + +public class TestCastToBlob { + + @Test + public void testNull() { + SQLite3Constant nullVal = SQLite3Constant.createNullConstant(); + SQLite3Constant castNullVal = SQLite3Cast.castToBlob(nullVal); + assertTrue(castNullVal.isNull()); + } + + @Test + public void testEmptyString() { + SQLite3Constant emptyBinary = SQLite3Constant.createTextConstant(""); + SQLite3Constant castVal = SQLite3Cast.castToBlob(emptyBinary); + assertArrayEquals(new byte[0], castVal.asBinary()); + } + + @Test + public void testString1() { + assertCastStringToBlob("0x12", "(BINARY) x'30783132'"); + } + + @Test + public void testString2() { + assertCastStringToBlob("123", "(BINARY) x'313233'"); + } + + void assertCastStringToBlob(String val, String expectedBlob) { + SQLite3Constant c = SQLite3Constant.createTextConstant(val); + SQLite3Constant binVal = SQLite3Cast.castToBlob(c); + assertEquals(binVal.toString(), expectedBlob); + } + +} diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToBoolean.java b/test/sqlancer/pqs/sqlite/cast/TestCastToBoolean.java new file mode 100644 index 000000000..ba6b9895a --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToBoolean.java @@ -0,0 +1,17 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.ast.SQLite3Constant; + +public class TestCastToBoolean { + + @Test + void nan() { + SQLite3Constant text = SQLite3Constant.createTextConstant("NaN"); + assertEquals(text.castToBoolean().asInt(), 0); + } + +} diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToInt.java b/test/sqlancer/pqs/sqlite/cast/TestCastToInt.java new file mode 100644 index 000000000..54f338ca3 --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToInt.java @@ -0,0 +1,89 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.ast.SQLite3Cast; +import sqlancer.sqlite3.ast.SQLite3Constant; + +class TestCastToInt { + + @Test + void test1() { + assertBinaryCastToInt("dbb25259", 0); + } + + @Test + void test2() { + assertBinaryCastToInt("d9a3", 0); + } + + @Test + void test3() { + assertCastStringToInt("1231231922047954197746780200000", Long.MAX_VALUE); + } + + @Test + void test4() { + assertCastStringToInt("1231231922047954197746780200000.5", Long.MAX_VALUE); + } + + @Test + void test5() { + assertCastStringToInt("-1231231922047954197746780200000.5", Long.MIN_VALUE); + } + + @Test + void testSign1() { + assertCastStringToInt("++123", 0); + } + + @Test + void testSign2() { + assertCastStringToInt("+123", 123); + } + + @Test + void testSign3() { + assertCastStringToInt("-123", -123); + } + + @Test + void testSign4() { + assertCastStringToInt("-+123", 0); + } + + @Test + void testSign5() { + assertCastStringToInt("+-123", 0); + } + + @Test + void testInfinity1() { + assertCastStringToInt("Infinity", 0); + } + + @Test + void testInfinity2() { + assertCastStringToInt("-Infinity", 0); + } + + @Test + void testNan() { + assertCastStringToInt("NaN", 0); + } + + void assertCastStringToInt(String val, long expectedLong) { + SQLite3Constant c = SQLite3Constant.createTextConstant(val); + SQLite3Constant intVal = SQLite3Cast.castToInt(c); + assertEquals(intVal.asInt(), expectedLong); + } + + void assertBinaryCastToInt(String val, long expectedLong) { + SQLite3Constant c = SQLite3Constant.createBinaryConstant(val); + SQLite3Constant intVal = SQLite3Cast.castToInt(c); + assertEquals(intVal.asInt(), expectedLong); + } + +} diff --git a/test/sqlancer/pqs/sqlite/cast/TestCastToNumeric.java b/test/sqlancer/pqs/sqlite/cast/TestCastToNumeric.java new file mode 100644 index 000000000..369e7390a --- /dev/null +++ b/test/sqlancer/pqs/sqlite/cast/TestCastToNumeric.java @@ -0,0 +1,117 @@ +package sqlancer.pqs.sqlite.cast; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Cast; +import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.schema.SQLite3DataType; + +class TestCastToNumeric { + + @Test + void testLong() { + long numbers[] = new long[] { 0, 1, 123, Long.MAX_VALUE, Long.MIN_VALUE }; + for (long number : numbers) { + assertEquals(castLongConstant(number), number); + } + } + + class StringTestTriple { + String value; + SQLite3DataType type; + Object expectedCastValue; + + public StringTestTriple(String value, SQLite3DataType type, Object expectedCastValue) { + this.value = value; + this.type = type; + this.expectedCastValue = expectedCastValue; + } + } + + @Test + void testString() { + List triples = new ArrayList<>(); + triples.add(new StringTestTriple("-3.0", SQLite3DataType.INT, -3L)); + triples.add(new StringTestTriple("8.2250617031974513E18", SQLite3DataType.REAL, 8.2250617031974513E18)); + + triples.add(new StringTestTriple("-2277224522334683278", SQLite3DataType.INT, -2277224522334683278L)); + + triples.add(new StringTestTriple("123a", SQLite3DataType.INT, 123L)); + triples.add(new StringTestTriple("", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("a", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("3", SQLite3DataType.INT, 3L)); + triples.add(new StringTestTriple("-3", SQLite3DataType.INT, -3L)); + triples.add(new StringTestTriple("0.0", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("+0", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("+9", SQLite3DataType.INT, 9L)); + triples.add(new StringTestTriple("++9", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("+-9", SQLite3DataType.INT, 0L)); + triples.add(new StringTestTriple("-1748799336", SQLite3DataType.INT, -1748799336L)); + triples.add(new StringTestTriple("-0", SQLite3DataType.INT, 0L)); + + triples.add(new StringTestTriple("4E ", SQLite3DataType.INT, 4L)); + triples.add(new StringTestTriple("3.0e+5", SQLite3DataType.INT, 300000L)); + triples.add(new StringTestTriple("-3.2", SQLite3DataType.REAL, -3.2d)); + triples.add(new StringTestTriple("10e9", SQLite3DataType.INT, 10000000000L)); + // triples.add(new StringTestTriple("-0.0", SQLite3DataType.REAL, 0.0d)); + triples.add(new StringTestTriple("9223372036854775807", SQLite3DataType.INT, 9223372036854775807L)); + triples.add(new StringTestTriple("4337561223119921152", SQLite3DataType.INT, 4337561223119921152L)); + triples.add(new StringTestTriple("7839344951195291815", SQLite3DataType.INT, 7839344951195291815L)); + + // infinities + triples.add(new StringTestTriple("-Infinity", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("Infinity", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("Inf", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("-Inf", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("NaN", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("1e500", SQLite3DataType.REAL, Double.POSITIVE_INFINITY)); // + triples.add(new StringTestTriple("-1e500", SQLite3DataType.REAL, Double.NEGATIVE_INFINITY)); // + + for (StringTestTriple triple : triples) { + SQLite3Constant castVal = SQLite3Cast.castToNumeric(SQLite3Constant.createTextConstant(triple.value)); + assertEquals(triple.expectedCastValue, castVal.getValue(), triple.value.toString()); + } + } + + @Test + void testBinary() { + List triples = new ArrayList<>(); + triples.add(new StringTestTriple("112B3980", SQLite3DataType.INT, 0L)); // +9� + triples.add(new StringTestTriple("0936", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0C36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0a36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0b36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0c36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0d36", SQLite3DataType.INT, 6L)); // 6 + triples.add(new StringTestTriple("0e36", SQLite3DataType.INT, 0L)); // 6 + triples.add(new StringTestTriple("1a347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("1b347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("1C32", SQLite3DataType.INT, 0L)); // FS2 + triples.add(new StringTestTriple("1D32", SQLite3DataType.INT, 0L)); // GS2 + triples.add(new StringTestTriple("1e32", SQLite3DataType.INT, 0L)); // RS2 + triples.add(new StringTestTriple("1f32", SQLite3DataType.INT, 0L)); // RS2 + triples.add(new StringTestTriple("2032", SQLite3DataType.INT, 2L)); // RS2 + triples.add(new StringTestTriple("09013454", SQLite3DataType.INT, 0L)); // RS2 + triples.add(new StringTestTriple("2016347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("2017347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("2018347C", SQLite3DataType.INT, 0L)); // + triples.add(new StringTestTriple("2019347C", SQLite3DataType.INT, 0L)); // + + for (StringTestTriple triple : triples) { + SQLite3Constant castVal = SQLite3Cast.castToNumeric( + SQLite3Constant.createBinaryConstant(SQLite3Visitor.hexStringToByteArray(triple.value))); + assertEquals(triple.expectedCastValue, castVal.getValue(), triple.value.toString()); + } + } + + private long castLongConstant(long constant) { + return SQLite3Cast.castToNumeric(SQLite3Constant.createIntConstant(constant)).asInt(); + } + +} diff --git a/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java new file mode 100644 index 000000000..b3ba57a10 --- /dev/null +++ b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java @@ -0,0 +1,24 @@ +package sqlancer.qpg.cockroachdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestCockroachDBQPG { + + @Test + public void testCockroachDBQPG() { + String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); + boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); + assumeTrue(cockroachDBIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, + "cockroachdb", "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java new file mode 100644 index 000000000..c858ee190 --- /dev/null +++ b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java @@ -0,0 +1,47 @@ +package sqlancer.qpg.cockroachdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.cockroachdb.CockroachDBOptions; +import sqlancer.cockroachdb.CockroachDBProvider; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; + +public class TestCockroachDBQueryPlan { + + @Test + void testCockroachDBQueryPlan() throws Exception { + String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); + boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); + assumeTrue(cockroachDBIsAvailable); + + String databaseName = "cockroachdb"; + CockroachDBProvider provider = new CockroachDBProvider(); + CockroachDBGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + CockroachDBOptions cockroachdbOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(cockroachdbOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals("left-join (cross);scan t2;select;scan t1;filters;filters (true);", queryPlan); + } + +} diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQPG.java b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java new file mode 100644 index 000000000..431468cee --- /dev/null +++ b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java @@ -0,0 +1,23 @@ +package sqlancer.qpg.materialize; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMaterializeQPG { + + @Test + public void testMaterializeQPG() { + String materialize = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materialize != null && materialize.equalsIgnoreCase("true"); + assumeTrue(materializeIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "--username", + "materialize", "materialize", "--oracle", "QUERY_PARTITIONING", "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java new file mode 100644 index 000000000..4d26bc08e --- /dev/null +++ b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java @@ -0,0 +1,49 @@ +package sqlancer.qpg.materialize; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeOptions; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; + +public class TestMaterializeQueryPlan { + + @Test + void testMaterializeQueryPlan() throws Exception { + String materialize = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materialize != null && materialize.equalsIgnoreCase("true"); + assumeTrue(materializeIsAvailable); + + String databaseName = "queryplan"; + MaterializeProvider provider = new MaterializeProvider(); + MaterializeGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + MaterializeOptions materializeOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(materializeOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals( + "With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2, #3, #0{c}) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2{c}) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", + queryPlan); + } + +} diff --git a/test/sqlancer/qpg/postgres/TestPostgresQPG.java b/test/sqlancer/qpg/postgres/TestPostgresQPG.java new file mode 100644 index 000000000..20ec3e91e --- /dev/null +++ b/test/sqlancer/qpg/postgres/TestPostgresQPG.java @@ -0,0 +1,22 @@ +package sqlancer.qpg.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestPostgresQPG { + + @Test + public void testPostgresQPG() { + String postgres = System.getenv("POSTGRES_AVAILABLE"); + boolean postgresIsAvailable = postgres != null && postgres.equalsIgnoreCase("true"); + assumeTrue(postgresIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "postgres", "postgres", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java new file mode 100644 index 000000000..c883e87d8 --- /dev/null +++ b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java @@ -0,0 +1,165 @@ +package sqlancer.qpg.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.PostgresProvider; + +public class TestPostgresQueryPlan { + + @Test + void testPostgresQueryPlan() throws Exception { + String databaseName = "postgres"; + PostgresProvider provider = new PostgresProvider(); + PostgresGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + PostgresOptions postgresOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(postgresOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + assertEquals("Nested Loop Seq Scan Materialize Seq Scan", queryPlan); + } + + @Test + void testFormatQueryPlan() throws Exception { + + PostgresProvider provider = new PostgresProvider(); + + String queryPlan = "[\n" + " {\n" + " \"Plan\": {\n" + " \"Node Type\": \"Aggregate\",\n" + + " \"Strategy\": \"Hashed\",\n" + " \"Partial Mode\": \"Simple\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 62998.82,\n" + " \"Total Cost\": 63009.32,\n" + + " \"Plan Rows\": 1050,\n" + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Planned Partitions\": 0,\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Append\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Startup Cost\": 27150.40,\n" + + " \"Total Cost\": 62996.20,\n" + " \"Plan Rows\": 1050,\n" + + " \"Plan Width\": 4,\n" + " \"Subplans Removed\": 0,\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Group\",\n" + + " \"Parent Relationship\": \"Member\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Startup Cost\": 27150.40,\n" + + " \"Total Cost\": 62949.08,\n" + " \"Plan Rows\": 200,\n" + + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Gather Merge\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 27150.40,\n" + " \"Total Cost\": 62948.08,\n" + + " \"Plan Rows\": 400,\n" + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + " \"Workers Planned\": 2,\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Group\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 26150.38,\n" + + " \"Total Cost\": 61901.89,\n" + " \"Plan Rows\": 200,\n" + + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Merge Join\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Join Type\": \"Inner\",\n" + + " \"Startup Cost\": 26150.38,\n" + + " \"Total Cost\": 56906.48,\n" + + " \"Plan Rows\": 1998164,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + + " \"Inner Unique\": false,\n" + + " \"Merge Cond\": \"(t0.c0 = t1.c0)\",\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Sort\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 25970.60,\n" + + " \"Total Cost\": 26362.39,\n" + + " \"Plan Rows\": 156719,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t0.c0\"],\n" + + " \"Sort Key\": [\"t0.c0\"],\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Seq Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": true,\n" + + " \"Async Capable\": false,\n" + + " \"Relation Name\": \"t0\",\n" + + " \"Schema\": \"public\",\n" + + " \"Alias\": \"t0\",\n" + + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 10301.95,\n" + + " \"Plan Rows\": 156719,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t0.c0\"],\n" + + " \"Filter\": \"(t0.c0 < 100)\"\n" + + " }\n" + " ]\n" + + " },\n" + " {\n" + + " \"Node Type\": \"Sort\",\n" + + " \"Parent Relationship\": \"Inner\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 179.78,\n" + + " \"Total Cost\": 186.16,\n" + + " \"Plan Rows\": 2550,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + + " \"Sort Key\": [\"t1.c0\"],\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Seq Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Relation Name\": \"t1\",\n" + + " \"Schema\": \"public\",\n" + + " \"Alias\": \"t1\",\n" + + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 35.50,\n" + + " \"Plan Rows\": 2550,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"]\n" + " }\n" + + " ]\n" + " }\n" + + " ]\n" + " }\n" + " ]\n" + + " }\n" + " ]\n" + " }\n" + " ]\n" + + " },\n" + " {\n" + " \"Node Type\": \"Bitmap Heap Scan\",\n" + + " \"Parent Relationship\": \"Member\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Relation Name\": \"t2\",\n" + + " \"Schema\": \"public\",\n" + " \"Alias\": \"t2\",\n" + + " \"Startup Cost\": 10.74,\n" + " \"Total Cost\": 31.37,\n" + + " \"Plan Rows\": 850,\n" + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t2.c0\"],\n" + " \"Recheck Cond\": \"(t2.c0 < 10)\",\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Bitmap Index Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Index Name\": \"t2_pkey\",\n" + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 10.53,\n" + " \"Plan Rows\": 850,\n" + + " \"Plan Width\": 0,\n" + " \"Index Cond\": \"(t2.c0 < 10)\"\n" + + " }\n" + " ]\n" + " }\n" + " ]\n" + " }\n" + + " ]\n" + " },\n" + " \"Planning Time\": 1.954\n" + " }\n" + "]\n"; + + String formatedQueryPlan = provider.formatQueryPlan(queryPlan); + assertEquals( + "Aggregate Append Group Bitmap Heap Scan Gather Merge Bitmap Index Scan Group Merge Join Sort Sort Seq Scan Seq Scan", + formatedQueryPlan); + } + +} diff --git a/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java b/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java new file mode 100644 index 000000000..488037237 --- /dev/null +++ b/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java @@ -0,0 +1,23 @@ +package sqlancer.qpg.sqlite; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestSQLiteQPG { + + @Test + public void testSqliteQPG() { + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", + "pqsdb" /* Workaround for connections not being closed */, "--qpg-enable", "true", "sqlite3", + "--oracle", "NoREC", "--test-fts", "false", "--test-rtree", "false", "--test-check-constraints", + "false", "--test-in-operator", "false" })); + } + +} diff --git a/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java b/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java new file mode 100644 index 000000000..7672f7edf --- /dev/null +++ b/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java @@ -0,0 +1,41 @@ +package sqlancer.qpg.sqlite; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3Options; +import sqlancer.sqlite3.SQLite3Provider; + +public class TestSQLiteQueryPlan { + + @Test + void testSQLiteQueryPlan() throws Exception { + String databaseName = "sqlite"; + SQLite3Provider provider = new SQLite3Provider(); + SQLite3GlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + SQLite3Options sqlite3Option = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(sqlite3Option); + state.setDatabaseName(databaseName); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals("SCAN t1;SCAN t2;RIGHT-JOIN t2;SCAN t2;", queryPlan); + } + +} diff --git a/test/sqlancer/qpg/tidb/TestTiDBQPG.java b/test/sqlancer/qpg/tidb/TestTiDBQPG.java new file mode 100644 index 000000000..66a43a41a --- /dev/null +++ b/test/sqlancer/qpg/tidb/TestTiDBQPG.java @@ -0,0 +1,24 @@ +package sqlancer.qpg.tidb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestTiDBQPG { + + @Test + public void testTiDBQPG() { + String tiDB = System.getenv("TIDB_AVAILABLE"); + boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); + assumeTrue(tiDBIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "tidb", + "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java b/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java new file mode 100644 index 000000000..d07bebe61 --- /dev/null +++ b/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java @@ -0,0 +1,49 @@ +package sqlancer.qpg.tidb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBOptions; +import sqlancer.tidb.TiDBProvider; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public class TestTiDBQueryPlan { + + @Test + void testTiDBQueryPlan() throws Exception { + String tiDB = System.getenv("TIDB_AVAILABLE"); + boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); + assumeTrue(tiDBIsAvailable); + + String databaseName = "tidb"; + TiDBProvider provider = new TiDBProvider(); + TiDBGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + TiDBOptions TiDBOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(TiDBOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals( + "HashJoin_7;TableReader_10(Build);Selection_9;TableFullScan_8;TableReader_12(Probe);TableFullScan_11;", + queryPlan); + } + +} diff --git a/test/sqlancer/reducer/TestASTBasedReducer.java b/test/sqlancer/reducer/TestASTBasedReducer.java new file mode 100644 index 000000000..549bbe84d --- /dev/null +++ b/test/sqlancer/reducer/TestASTBasedReducer.java @@ -0,0 +1,267 @@ +package sqlancer.reducer; + +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import org.junit.jupiter.api.Test; +import sqlancer.common.query.Query; + +import java.util.List; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestASTBasedReducer { + @Test + void testRemovingClauses() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStr = { + "SELECT DISTINCT * FROM v0 WHERE ((v0.rowid || ( (v0.c + v0.d) < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("&&"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + String outcome = TestEnvironment.getQueriesString(reducedResult); + assertEquals(outcome, "SELECT * FROM v0 WHERE v0.c && v0.c;"); + } + + @Test + void testReducingMultipleTokensToOne() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStr = { + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("||"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals(TestEnvironment.getQueriesString(reducedResult), "SELECT row_id FROM v0 WHERE v0.rowid || 0;"); + } + + @Test + void testMultipleStatements() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + + return queriesString.toUpperCase().contains("UNION"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals(TestEnvironment.getQueriesString(reducedResult), + "SELECT row_id FROM v0;\nSELECT row_id FROM v0 UNION SELECT * FROM v0;"); + } + + @Test + void removeJoins() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { "SELECT * FROM t0, t1, t2, t3, t4 Where t2.val = t1.val" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + String queriesString = TestEnvironment.getQueriesString(statements); + if (!queriesString.contains("WHERE")) { + return false; + } + String[] split = queriesString.split("WHERE"); + String columns = split[0]; + String condition = split[1]; + return columns.contains("t1") && condition.contains("t1"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals(TestEnvironment.getQueriesString(reducedResult), "SELECT * FROM t0, t1 WHERE t1.val;"); + } + + @Test + void testComplicated() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT STRING_AGG(v0.c2) FROM t0, v0 WHERE (CASE true WHEN (ABS(true) BETWEEN (v0.c0 LIKE NULL ESCAPE v0.c2) AND (DATE '1970-01-23' NOT IN (v0.c2))) THEN (0.07914839711718646 NOT BETWEEN '' AND ((v0.c0)OR(v0.c2))) WHEN v0.c1 THEN ((v0.c1)-(v0.c0)) WHEN t0.c1 THEN (TIMESTAMP '1969-12-29 20:22:33' IN (PI(), v0.c2, (v0.c1 BETWEEN '' AND v0.rowid))) WHEN v0.c1 THEN TIMESTAMP '1969-12-16 17:24:43' WHEN ((((v0.c1)-(t0.c0)))||(t0.c0)) THEN true ELSE ((0.279978719843174)/(((v0.c1)>(DATE '1969-12-19')))) END ) GROUP BY ((DATE '1970-01-24') IS NULL), t0.c1, (CASE (v0.c1 LIKE ((0.9833120083624495)SIMILAR TO(t0.rowid)) ESCAPE CEIL(TIMESTAMP '1970-01-11 16:38:26')) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END );" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + Function>, Boolean> condition = statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.toUpperCase().contains("CASE"); + }; + env.setBugInducingCondition(condition); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertTrue(condition.apply(reducedResult)); + } + + @Test + void testSimplifyingConstantStringValue() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { "SELECT * FROM t0 WHERE v LIKE '[vQ3㭫oQ';" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("LIKE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("SELECT * FROM t0 WHERE v LIKE '_';", TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testSimplifyingConstantLongValue() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { "SELECT * FROM t0 where t0.v = 314598267;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("t0.v = "); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("SELECT * FROM t0 WHERE t0.v = 0;", TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testSubSelects() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT AVG(c0) FROM (SELECT SUM(c1) AS c0 FROM t1 GROUP BY c2 LIMIT 32 OFFSET 128) AS t1;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("AVG"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("SELECT AVG(c0) FROM (SELECT SUM(c1) AS c0 FROM t1) AS t1;", + TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testInsert() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "INSERT INTO t1(c2, c0) VALUES (1508438260, 2929), (1508438260, TIMESTAMP '1969-12-26 01:57:21'), (0.5347171705591047, 398662142);" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("(0.5347171705591047, 398662142)"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("INSERT INTO t1 (c2, c0) VALUES (0.5347171705591047, 398662142);", + TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testWithSelect() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "WITH cte1 AS (SELECT a, b FROM table1 where a < b), cte2 AS (SELECT c, d FROM table2 where c = d) SELECT b, d FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("table1"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("WITH cte1 AS (SELECT a FROM table1) SELECT b FROM cte1;", + TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testRoundDouble() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + String[] queriesStrs = { "SELECT * FROM t0 WHERE (2.1427572639 IS NULL);" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("WHERE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("SELECT * FROM t0 WHERE 2.143 IS NULL;", TestEnvironment.getQueriesString(reducedResult)); + } + +} diff --git a/test/sqlancer/reducer/TestEnvironment.java b/test/sqlancer/reducer/TestEnvironment.java new file mode 100644 index 000000000..7c1155490 --- /dev/null +++ b/test/sqlancer/reducer/TestEnvironment.java @@ -0,0 +1,139 @@ +package sqlancer.reducer; + +import sqlancer.*; +import sqlancer.common.query.Query; +import sqlancer.reducer.VirtualDB.VirtualDBGlobalState; +import sqlancer.reducer.VirtualDB.VirtualDBProvider; +import sqlancer.reducer.VirtualDB.VirtualDBQuery; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.ServiceLoader; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * TODO: Make Connection a generic type OR Fake a conn QUERY AND CONNECTION BOTH ARE FAKE. FAKE QUERY sub class + */ +public class TestEnvironment { + private final String databaseName = "virtual_db"; + private final MainOptions options = new MainOptions(); + private VirtualDBProvider provider = null; + private VirtualDBGlobalState state, newGlobalState; + + private Reducer reducer = null; + + enum ReducerType { + USING_STATEMENT_REDUCER, USING_AST_BASED_REDUCER + }; + + private TestEnvironment(ReducerType type) throws Exception { + setUpTestingEnvironment(); + if (type == ReducerType.USING_STATEMENT_REDUCER) { + reducer = new StatementReducer<>(provider); + } else if (type == ReducerType.USING_AST_BASED_REDUCER) { + reducer = new ASTBasedReducer<>(provider); + } + } + + public static TestEnvironment getStatementReducerEnv() throws Exception { + return new TestEnvironment(ReducerType.USING_STATEMENT_REDUCER); + } + + public static TestEnvironment getASTBasedReducerEnv() throws Exception { + return new TestEnvironment(ReducerType.USING_AST_BASED_REDUCER); + } + + /** + * @param queries: + * List of Query + * + * @return String of queries that appended together with '\n' separated (no '\n' at the last line) + */ + public static String getQueriesString(List> queries) { + return queries.stream().map(Query::getQueryString).collect(Collectors.joining("\n")); + } + + private VirtualDBGlobalState createGlobalState() { + try { + return provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @SuppressWarnings("rawtypes") + private void initVirtualDBProvider() { + try { + ServiceLoader loader = ServiceLoader.load(DatabaseProvider.class); + for (DatabaseProvider provider : loader) { + if (provider.getDBMSName().equals(databaseName)) { + this.provider = (VirtualDBProvider) provider; + break; + } + } + if (provider == null) { + throw new AssertionError("testing provider not registered"); + } + } catch (Exception e) { + throw new AssertionError(e); + } + + } + + private void setUpTestingEnvironment() throws Exception { + initVirtualDBProvider(); + state = createGlobalState(); + StateToReproduce stateToReproduce = provider.getStateToReproduce(databaseName); + + state.setState(stateToReproduce); + state.setDatabaseName(databaseName); + // A really hacky way to enable reducer... + Field field = options.getClass().getDeclaredField("useReducer"); + field.setAccessible(true); + field.set(options, true); + state.setMainOptions(options); + + // Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + // state.setStateLogger(logger); + + try (SQLConnection con = provider.createDatabase(state)) { + state.setConnection(con); + newGlobalState = createGlobalState(); + Main.StateLogger newLogger = new Main.StateLogger(databaseName, provider, options); + newGlobalState.setStateLogger(newLogger); + state.setStateLogger(newLogger); + newGlobalState.setState(stateToReproduce); + newGlobalState.setDatabaseName(databaseName); + newGlobalState.setMainOptions(options); + } + } + + public void setInitialStatementsFromStrings(List statements) { + List> queries = new ArrayList<>(); + for (String s : statements) { + queries.add(new VirtualDBQuery(s)); + } + state.getState().setStatements(queries); + } + + public void setBugInducingCondition(Function>, Boolean> bugInducingCondition) { + state.setBugInducingCondition(bugInducingCondition); + newGlobalState.setBugInducingCondition(bugInducingCondition); + } + + public void runReduce() throws Exception { + + Reproducer reproducer = provider.generateAndTestDatabase(newGlobalState); + reducer.reduce(state, reproducer, newGlobalState); + } + + public List> getReducedStatements() { + return newGlobalState.getState().getStatements(); + } + + public List> getInitialStatements() { + return state.getState().getStatements(); + } +} diff --git a/test/sqlancer/reducer/TestStatementReducer.java b/test/sqlancer/reducer/TestStatementReducer.java new file mode 100644 index 000000000..d9e364c84 --- /dev/null +++ b/test/sqlancer/reducer/TestStatementReducer.java @@ -0,0 +1,82 @@ +package sqlancer.reducer; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.common.query.Query; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestStatementReducer { + + @Test + void testSimple() throws Exception { + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); + + String[] queriesStr = { "CREATE TABLE FAKE_TABLE;", "SELECT * FROM FAKE_TABLE;", "EXIT", }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("SELECT"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals(1, reducedResult.size()); + assertEquals("SELECT * FROM FAKE_TABLE;", reducedResult.get(0).toString()); + + } + + @Test + void testDeltaDebugging() throws Exception { + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); + List fakeStatements = new ArrayList<>(); + for (int i = 0; i < 10000; i++) { + String statement = "Statement_" + i + ";"; + fakeStatements.add(statement); + } + + env.setInitialStatementsFromStrings(fakeStatements); + env.setBugInducingCondition(statements -> { + String queries = TestEnvironment.getQueriesString(statements); + return queries.contains("Statement_29;"); + }); + + env.runReduce(); + List> reducedQueries = env.getReducedStatements(); + String queriesString = TestEnvironment.getQueriesString(reducedQueries); + assertEquals(queriesString, "Statement_29;"); + } + + @Test + void testDeltaDebuggingWithStatementsCombination() throws Exception { + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); + List fakeStatements = new ArrayList<>(); + + String pattern = "(.*\\n)*(Statement_2;)\\n(.*\\n)*(Statement_318);\\n(.*\\n)*(Statement_990;)(.*\\n)*.*"; + for (int i = 0; i < 1000; i++) { + String statement = "Statement_" + i + ";"; + fakeStatements.add(statement); + } + + env.setInitialStatementsFromStrings(fakeStatements); + env.setBugInducingCondition(queryList -> { + String queries = TestEnvironment.getQueriesString(queryList); + return Pattern.matches(pattern, queries); + }); + + env.runReduce(); + List> reducedQueries = env.getReducedStatements(); + String queriesString = TestEnvironment.getQueriesString(reducedQueries); + assertEquals(queriesString, "Statement_2;\nStatement_318;\nStatement_990;"); + } + + @Test + void testSQLite3WithStatementReducer() { + Main.executeMain(new String[] { "--random-seed", "0", "--use-reducer", "--timeout-seconds", "60", + "--num-threads", "4", "sqlite3", "--oracle", "NoREC" }); + } + +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java b/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java new file mode 100644 index 000000000..6a562a90d --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java @@ -0,0 +1,18 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.SQLConnection; + +import java.sql.Connection; +import java.sql.SQLException; + +public class VirtualDBConnection extends SQLConnection { + + public VirtualDBConnection(Connection connection) { + super(connection); + } + + @Override + public void close() throws SQLException { + + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java b/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java new file mode 100644 index 000000000..64c783c11 --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java @@ -0,0 +1,12 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.common.query.ExpectedErrors; + +public final class VirtualDBErrors { + public VirtualDBErrors() { + } + + public static void addErrors(ExpectedErrors errors) { + errors.add("Default error"); + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java b/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java new file mode 100644 index 000000000..a0548495c --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java @@ -0,0 +1,56 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.Query; + +import java.util.List; +import java.util.function.Function; + +@SuppressWarnings("all") +public class VirtualDBGlobalState extends SQLGlobalState { + + private SQLConnection virtualConn = new SQLConnection(null); + private StringBuilder queriesStringBuilder = new StringBuilder(); + private Function>, Boolean> bugInducingCondition = null; + + public Function>, Boolean> getBugInducingCondition() { + return bugInducingCondition; + } + + public void setBugInducingCondition(Function>, Boolean> condition) { + bugInducingCondition = (condition); + } + + @Override + protected VirtualDBSchema readSchema() throws Exception { + return null; + } + + @Override + public SQLConnection getConnection() { + // It's a fake engine, so the connection would not be available :) + return virtualConn; + } + + @Override + public void setConnection(SQLConnection con) { + // A fake connection could also not be closed. + // So nothing would be done here. + // And reset the query String (Seems needless) + // queriesStringBuilder = new StringBuilder(); + } + + // public String getCurrentQueriesString() { + // return queriesStringBuilder.toString(); + // } + + @Override + public boolean executeStatement(Query q, String... fills) throws Exception { + if (queriesStringBuilder.length() != 0) { + queriesStringBuilder.append("\n"); + } + queriesStringBuilder.append(q.getQueryString()); + return true; + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java b/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java new file mode 100644 index 000000000..72cbe918c --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java @@ -0,0 +1,29 @@ +package sqlancer.reducer.VirtualDB; + +import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.reducer.VirtualDB.VirtualDBOptions.VirtualDBFactory; + +import java.util.ArrayList; +import java.util.List; + +@Parameters(separators = "=", commandDescription = "VirtualDB (default port: " + "-1" + ", default host: " + "127.0.0.1" + + ")") +public class VirtualDBOptions implements DBMSSpecificOptions { + + List factories = new ArrayList<>(); + + @Override + public List getTestOracleFactory() { + return factories; + } + + public static class VirtualDBFactory implements OracleFactory { + @Override + public TestOracle create(VirtualDBGlobalState globalState) throws Exception { + return null; + } + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java b/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java new file mode 100644 index 000000000..697e5c60e --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java @@ -0,0 +1,54 @@ +package sqlancer.reducer.VirtualDB; + +import com.google.auto.service.AutoService; +import sqlancer.DatabaseProvider; +import sqlancer.Reproducer; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; + +@AutoService(DatabaseProvider.class) +public class VirtualDBProvider extends SQLProviderAdapter { + + private Reproducer reproducerForTesting; + + public VirtualDBProvider() { + super(VirtualDBGlobalState.class, VirtualDBOptions.class); + } + + @Override + public SQLConnection createDatabase(VirtualDBGlobalState globalState) throws Exception { + return new VirtualDBConnection(null); + } + + @Override + public String getDBMSName() { + return "virtual_db"; + } + + @Override + public void generateDatabase(VirtualDBGlobalState globalState) throws Exception { + + } + + @Override + public Reproducer generateAndTestDatabase(VirtualDBGlobalState globalState) throws Exception { + return state -> { + if (globalState.getBugInducingCondition() == null) + return false; + return globalState.getBugInducingCondition().apply(globalState.getState().getStatements()); + }; + } + + @Override + public Class getGlobalStateClass() { + return super.getGlobalStateClass(); + } + + public Reproducer getReproducerForTesting() { + return reproducerForTesting; + } + + public void setReproducerForTesting(Reproducer reproducer) { + this.reproducerForTesting = reproducer; + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java new file mode 100644 index 000000000..9767723e0 --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java @@ -0,0 +1,31 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; + +import java.sql.SQLException; + +public class VirtualDBQuery extends SQLQueryAdapter { + private static final long serialVersionUID = 1L; + + public VirtualDBQuery(String query) { + // Since the base class must check the format + // We judge if the statement could affect schema. A bit hacky tho. + super(query, (query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN"))); + } + + public VirtualDBQuery(String query, boolean couldAffectSchema) { + super(query, couldAffectSchema); + } + + @Override + public > boolean execute(G globalState, String... fills) + throws SQLException { + try { + return globalState.executeStatement(this, fills); + } catch (Exception e) { + throw new SQLException(e); + } + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java b/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java new file mode 100644 index 000000000..9ea5b422e --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java @@ -0,0 +1,46 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.reducer.VirtualDB.VirtualDBSchema.VirtualDBTable; + +import java.util.List; + +public class VirtualDBSchema extends AbstractSchema { + + public VirtualDBSchema(List databaseTables) { + super(databaseTables); + } + + public static class VirtualDBTable extends AbstractTable { + protected VirtualDBTable(String name, List columns, List indexes, + boolean isView) { + super(name, columns, indexes, isView); + } + + @Override + public long getNrRows(VirtualDBGlobalState globalState) { + return 0; + } + } + + public static class VirtualDBIndex extends TableIndex { + + protected VirtualDBIndex(String indexName) { + super(indexName); + } + } + + public static class VirtualDBDataType { + + } + + public static class VirtualDBColumn extends AbstractTableColumn { + + public VirtualDBColumn(String name, VirtualDBTable table, VirtualDBDataType type) { + super(name, table, type); + } + } +}