From c24fc250c57cbf39a873481bd702225b9392100f Mon Sep 17 00:00:00 2001 From: Aleksei Bukhalov Date: Mon, 13 Jul 2026 13:36:08 +0000 Subject: [PATCH] mariadb-duckdb: switch to Parquet, remove PRIMARY KEY, add 2026-07-13 results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch dataset ingestion from TSV to Parquet via run_in_duckdb/read_parquet - Remove PRIMARY KEY → no ART index overhead on load - start/stop: use mariadbd-safe / mariadb-admin instead of systemctl - start: cap duckdb_max_threads=nproc/2 on machines with >32 logical CPUs (empirically optimal at physical core count; HT siblings add overhead) Results (CI build 70621, no PRIMARY KEY): c6a.4xlarge (16 vCPU): load=100 s, concurrent QPS=1.530 c6a.metal (192 vCPU): load=125 s, concurrent QPS=13.947 --- mariadb-duckdb/benchmark.sh | 4 +- mariadb-duckdb/create.sql | 4 +- mariadb-duckdb/install | 9 +-- mariadb-duckdb/load | 26 ++++---- .../results/20260713/c6a.4xlarge.json | 59 +++++++++++++++++++ .../results/20260713/c6a.metal.json | 59 +++++++++++++++++++ mariadb-duckdb/start | 21 ++++++- mariadb-duckdb/stop | 2 +- 8 files changed, 160 insertions(+), 24 deletions(-) create mode 100644 mariadb-duckdb/results/20260713/c6a.4xlarge.json create mode 100644 mariadb-duckdb/results/20260713/c6a.metal.json diff --git a/mariadb-duckdb/benchmark.sh b/mariadb-duckdb/benchmark.sh index 124a77209b..0225254a25 100755 --- a/mariadb-duckdb/benchmark.sh +++ b/mariadb-duckdb/benchmark.sh @@ -1,3 +1,5 @@ #!/bin/bash -export BENCH_DOWNLOAD_SCRIPT="download-hits-tsv" +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +export BENCH_DURABLE=yes +export BENCH_RESTARTABLE=yes exec ../lib/benchmark-common.sh diff --git a/mariadb-duckdb/create.sql b/mariadb-duckdb/create.sql index 5ca83246fe..47415b5f9f 100644 --- a/mariadb-duckdb/create.sql +++ b/mariadb-duckdb/create.sql @@ -1,3 +1,4 @@ +set global duckdb_require_primary_key = OFF; CREATE TABLE hits ( WatchID BIGINT NOT NULL, @@ -104,6 +105,5 @@ CREATE TABLE hits HasGCLID SMALLINT NOT NULL, RefererHash BIGINT NOT NULL, URLHash BIGINT NOT NULL, - CLID INTEGER NOT NULL, - PRIMARY KEY (CounterID, EventDate, UserID, EventTime, WatchID) + CLID INTEGER NOT NULL ) ENGINE=DuckDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/mariadb-duckdb/install b/mariadb-duckdb/install index 8baf9c88f5..3d980241df 100755 --- a/mariadb-duckdb/install +++ b/mariadb-duckdb/install @@ -1,7 +1,7 @@ #!/bin/bash set -eu -CI_SOURCES_URL="https://ci.mariadb.org/68929/amd64-ubuntu-2404-deb-autobake/mariadb.sources" +CI_SOURCES_URL="https://ci.mariadb.org/70621/amd64-ubuntu-2404-deb-autobake/mariadb.sources" # Idempotent: if DuckDB plugin is already active, nothing to do. if sudo mariadb -e "SELECT PLUGIN_STATUS FROM information_schema.PLUGINS WHERE PLUGIN_NAME='DuckDB';" 2>/dev/null | grep -q 'ACTIVE'; then @@ -19,19 +19,16 @@ if dpkg -l | grep -qE '^ii\s+(mariadb-server|mysql-server)'; then sudo rm -rf /var/lib/mysql /etc/mysql fi -# Add MariaDB CI build as apt source (Trusted: yes — unsigned CI packages, testing only). sudo wget -q -O /etc/apt/sources.list.d/mariadb-duckdb-ci.sources "$CI_SOURCES_URL" - sudo apt-get update -y -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - mariadb-server mariadb-client +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mariadb-server mariadb-client -# Configure the DuckDB storage engine plugin (from mariadb.org article defaults). sudo tee /etc/mysql/mariadb.conf.d/duckdb.cnf >/dev/null <<'EOF' [mysqld] plugin-maturity=alpha plugin-load-add=ha_duckdb.so duckdb-memory-limit=24G +duckdb-allow-run-in-duckdb=ON EOF sudo systemctl enable mariadb diff --git a/mariadb-duckdb/load b/mariadb-duckdb/load index bcda9c407e..2d8354ae6e 100755 --- a/mariadb-duckdb/load +++ b/mariadb-duckdb/load @@ -13,22 +13,26 @@ sudo mariadb -e "DROP DATABASE IF EXISTS test" sudo mariadb -e "CREATE DATABASE test" sudo mariadb test < create.sql -# Load via DuckDB's native vectorized CSV reader (12x faster than LOAD DATA LOCAL +# Load via DuckDB's native vectorized parquet reader (faster than LOAD DATA LOCAL # INFILE which routes every row through MariaDB's single-threaded write_row path). -# hits.tsv is tab-delimited with datetime strings in TIMESTAMP columns, matching -# the table schema — no type conversion needed. # # DuckDB glob expander requires read permission on all parent directories. # /home/ubuntu is mode 751 (o+x only) so DuckDB can traverse but not list it, # causing "No files found" errors. Hardlink to /var/lib/mysql/ (mysql-owned, -# mode 755) resolves this instantly without copying 70 GB. -HITS_LINK=/var/lib/mysql/hits_load.tsv -sudo ln "$(realpath hits.tsv)" "$HITS_LINK" -sudo chmod 644 "$HITS_LINK" -# NULLSTR '\N' matches MySQL/MariaDB convention: only \N is NULL, empty fields → ''. -# Without this, DuckDB treats every empty TSV field as NULL → NOT NULL violations. -duck "COPY test.hits FROM '$HITS_LINK' (FORMAT CSV, DELIMITER '\t', NULLSTR '\N')" +# mode 755) resolves this instantly without copying 14 GB. +HITS_LINK=/var/lib/mysql/hits.parquet +if [ ! -f "$HITS_LINK" ]; then + sudo ln "$(realpath hits.parquet)" "$HITS_LINK" + sudo chmod 644 "$HITS_LINK" +fi +duck "INSERT INTO test.hits +SELECT * REPLACE ( + make_date(EventDate) AS EventDate, + epoch_ms(EventTime * 1000) AS EventTime, + epoch_ms(ClientEventTime * 1000) AS ClientEventTime, + epoch_ms(LocalEventTime * 1000) AS LocalEventTime) +FROM read_parquet('$HITS_LINK', binary_as_string=True);" sudo rm -f "$HITS_LINK" -rm -f hits.tsv +rm -f hits.parquet sync diff --git a/mariadb-duckdb/results/20260713/c6a.4xlarge.json b/mariadb-duckdb/results/20260713/c6a.4xlarge.json new file mode 100644 index 0000000000..95157f9de6 --- /dev/null +++ b/mariadb-duckdb/results/20260713/c6a.4xlarge.json @@ -0,0 +1,59 @@ +{ + "system": "MariaDB (DuckDB)", + "date": "2026-07-13", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++", "column-oriented", "MySQL compatible", "DuckDB derivative"], + "load_time": 100.378, + "data_size": 20458778624, + "concurrent_qps": 1.530, + "concurrent_error_ratio": 0.000, + "result": [ + [0.039, 0.002, 0.001], + [0.145, 0.005, 0.005], + [0.245, 0.021, 0.021], + [0.316, 0.032, 0.032], + [0.503, 0.301, 0.299], + [1.004, 0.258, 0.261], + [0.114, 0.003, 0.003], + [0.163, 0.007, 0.007], + [0.900, 0.378, 0.375], + [1.613, 0.528, 0.525], + [0.541, 0.072, 0.071], + [0.782, 0.082, 0.080], + [0.986, 0.333, 0.332], + [2.308, 0.655, 0.677], + [1.391, 0.382, 0.386], + [0.548, 0.341, 0.340], + [2.219, 0.779, 0.790], + [1.977, 0.524, 0.532], + [4.615, 2.761, 2.781], + [0.221, 0.005, 0.005], + [10.688, 0.484, 0.485], + [12.324, 0.500, 0.498], + [14.490, 1.657, 2.319], + [0.596, 0.068, 0.066], + [0.216, 0.021, 0.024], + [0.922, 0.103, 0.102], + [0.232, 0.021, 0.020], + [10.865, 0.404, 0.405], + [8.675, 6.651, 6.601], + [0.201, 0.027, 0.026], + [2.796, 0.287, 0.285], + [5.822, 0.373, 0.380], + [5.005, 1.663, 1.750], + [11.170, 1.654, 1.649], + [11.192, 1.852, 1.827], + [0.610, 0.428, 0.431], + [0.134, 0.029, 0.026], + [0.122, 0.010, 0.009], + [0.121, 0.015, 0.015], + [0.229, 0.054, 0.064], + [0.136, 0.006, 0.006], + [0.148, 0.006, 0.006], + [0.238, 0.150, 0.154] + ] +} diff --git a/mariadb-duckdb/results/20260713/c6a.metal.json b/mariadb-duckdb/results/20260713/c6a.metal.json new file mode 100644 index 0000000000..7f832497ec --- /dev/null +++ b/mariadb-duckdb/results/20260713/c6a.metal.json @@ -0,0 +1,59 @@ +{ + "system": "MariaDB (DuckDB)", + "date": "2026-07-13", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++", "column-oriented", "MySQL compatible", "DuckDB derivative"], + "load_time": 125.173, + "data_size": 20562063360, + "concurrent_qps": 13.947, + "concurrent_error_ratio": 0.000, + "result": [ + [0.035, 0.002, 0.002], + [0.099, 0.005, 0.004], + [0.140, 0.006, 0.006], + [0.523, 0.008, 0.007], + [0.556, 0.067, 0.064], + [1.663, 0.055, 0.054], + [0.089, 0.002, 0.002], + [0.094, 0.006, 0.006], + [1.406, 0.080, 0.077], + [2.424, 0.117, 0.115], + [1.104, 0.025, 0.024], + [1.449, 0.026, 0.026], + [1.673, 0.063, 0.059], + [3.163, 0.116, 0.108], + [2.226, 0.068, 0.065], + [0.574, 0.073, 0.074], + [3.156, 0.147, 0.139], + [3.177, 0.146, 0.147], + [5.910, 0.446, 0.445], + [0.105, 0.006, 0.005], + [15.024, 0.087, 0.082], + [17.363, 0.082, 0.077], + [17.764, 0.695, 0.666], + [1.129, 0.028, 0.027], + [0.180, 0.014, 0.012], + [1.661, 0.028, 0.041], + [0.170, 0.014, 0.011], + [15.271, 0.074, 0.067], + [11.911, 1.083, 1.059], + [0.103, 0.014, 0.012], + [4.245, 0.059, 0.059], + [7.893, 0.135, 0.164], + [5.652, 0.364, 0.360], + [15.121, 0.311, 0.329], + [15.130, 0.321, 0.327], + [0.205, 0.101, 0.100], + [0.106, 0.026, 0.032], + [0.083, 0.008, 0.009], + [0.125, 0.013, 0.012], + [0.199, 0.053, 0.052], + [0.108, 0.008, 0.006], + [0.111, 0.008, 0.006], + [0.207, 0.130, 0.130] + ] +} diff --git a/mariadb-duckdb/start b/mariadb-duckdb/start index bc0d6cce8e..39e702cff8 100755 --- a/mariadb-duckdb/start +++ b/mariadb-duckdb/start @@ -1,7 +1,22 @@ #!/bin/bash set -eu -if sudo mariadb -e "SELECT 1" >/dev/null 2>&1; then - exit 0 +if ! sudo mariadb -e "SELECT 1" >/dev/null 2>&1; then + /bin/sh /usr/bin/mariadbd-safe --datadir=/var/lib/mysql &>/dev/null & + # Wait up to 30s for the server to accept connections. + for _ in $(seq 1 30); do + sudo mariadb -e "SELECT 1" >/dev/null 2>&1 && break + sleep 1 + done + if ! sudo mariadb -e "SELECT 1" >/dev/null 2>&1; then + echo "mariadbd-safe did not accept connections within 30s" >&2 + exit 1 + fi +fi + +# Cap at physical core count on HT machines: above nproc/2 DuckDB gains no +# throughput from hyperthreaded siblings but pays extra scheduling overhead. +CORES=$(nproc) +if [ "$CORES" -gt 32 ]; then + sudo mariadb -e "SET GLOBAL duckdb_max_threads = $((CORES / 2))" fi -sudo systemctl start mariadb diff --git a/mariadb-duckdb/stop b/mariadb-duckdb/stop index 1b523d374d..175e1f110c 100755 --- a/mariadb-duckdb/stop +++ b/mariadb-duckdb/stop @@ -1,2 +1,2 @@ #!/bin/bash -sudo systemctl stop mariadb || true +/usr/bin/mariadb-admin shutdown