diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 0000000..131fb70 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,8 @@ +[source.crates-io] +replace-with = 'tuna' + +[source.tuna] +registry = "https://mirrors.tuna.tsinghua.edu.cn/git/crates.io-index.git" + +# [target.x86_64-unknown-linux-musl] +# linker = "musl-gcc" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index ccaf49d..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: CI - -on: - push: - branches: [master] - pull_request: - branches: [master] - -env: - CARGO_TERM_COLOR: always - RUSTFLAGS: -D warnings - -jobs: - check: - name: Check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - run: cargo check --workspace - - test: - name: Test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - run: cargo test --workspace - - clippy: - name: Clippy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - uses: Swatinem/rust-cache@v2 - - run: cargo clippy --workspace -- -D warnings - - fmt: - name: Format - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - run: cargo fmt --all -- --check - - build-release: - name: Build Release - runs-on: ubuntu-latest - needs: [check, test, clippy, fmt] - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - run: cargo build --release --workspace - - uses: actions/upload-artifact@v4 - with: - name: ttstack-linux-x86_64 - path: | - target/release/tt - target/release/tt-ctl - target/release/tt-agent diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..1fe0967 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,22 @@ +stages: + - build + - test + - deploy + +job_build: + stage: build + script: + - make lint + tags: + - rust + +job_test: + stage: test + script: + - make test + - cargo kcov --all + - COVERAGE=$(grep -Po '(?<="covered":")\d+(?:\.\d+)?(?=")' target/cov/index.js) + # Tips: use '^Coverage:\s+(\d+(?:\.\d+)?)' on gitlab to capture the result + - echo "Coverage:" ${COVERAGE} + tags: + - rust diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..7162efb --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tools/firecracker"] + path = tools/firecracker + url = https://gitee.com/kt10/firecracker diff --git a/Cargo.toml b/Cargo.toml index 4764c46..d90be51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,34 +1,25 @@ [workspace] -resolver = "3" members = [ - "crates/core", - "crates/agent", - "crates/ctl", - "crates/cli", + "src/core_def", + "src/core", + "src/server_def", + "src/server", + "src/proxy", + "src/client", + "src/rexec", + "src/utils", ] -[workspace.package] -version = "0.5.0" -edition = "2024" -authors = ["fh "] -license = "MIT" - -[workspace.dependencies] -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -ruc = { version = "9.3", features = ["cmd"] } -rusqlite = { version = "0.35", features = ["bundled"] } -tokio = { version = "1", features = ["full"] } -axum = "0.8" -reqwest = { version = "0.12", features = ["json"] } -clap = { version = "4", features = ["derive", "env"] } -uuid = { version = "1", features = ["v4"] } -nix = { version = "0.29", features = ["net", "socket", "ioctl", "fs", "signal"] } -tempfile = "3" -toml = "0.8" +[profile.dev] +overflow-checks = true +panic = "unwind" [profile.release] lto = true incremental = false overflow-checks = false panic = "unwind" + +[profile.bench] +codegen-units = 1 +overflow-checks = false diff --git a/LICENSE b/LICENSE index 313473e..a0aa749 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 fh +Copyright (c) 2020 范辉 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 06b032b..5da5f50 100644 --- a/Makefile +++ b/Makefile @@ -1,98 +1,108 @@ -PREFIX ?= /opt/ttstack -CARGO ?= cargo -CARGO_FLAG ?= -.PHONY: all build release test lint fmt fmt-check doc clean \ - install uninstall deploy-agent deploy-ctl deploy deploy-dist help +CARGO = ~/.cargo/bin/cargo +BUILD_DIR = /tmp/.__tt_build_dir__ +PACK_NAME = tt +PACK_DIR = $(BUILD_DIR)/$(PACK_NAME) +TARGET = $(shell rustup toolchain list | grep default | sed 's/^[^-]\+-//' | sed 's/ \+(default).*//') -all: fmt lint build test +all: pack -## Build debug binaries build: - $(CARGO) build $(CARGO_FLAG) + $(CARGO) build --bins -## Build optimized release binaries release: - $(CARGO) build --release $(CARGO_FLAG) + $(CARGO) build --bins --release + +pack: release + -@ rm -rf $(PACK_DIR) + mkdir -p $(PACK_DIR) + cd target/release && cp tt ttserver ttproxy $(PACK_DIR)/ + cp tools/install.sh $(PACK_DIR)/ + \ + git submodule update --init --recursive + cd tools/firecracker \ + && cargo build --release --target-dir=$(BUILD_DIR) \ + && cp $(BUILD_DIR)/$(TARGET)/release/firecracker $(PACK_DIR)/ + \ + chmod -R +x $(PACK_DIR) + tar -C $(BUILD_DIR) -zcf $(PACK_NAME).tar.gz $(PACK_NAME) + +install: + $(CARGO) install -f --bins --path src/rexec --root=/usr/local/ + $(CARGO) install -f --bins --path src/client --root=/usr/local/ + $(CARGO) install -f --bins --path src/server --root=/usr/local/ # will fail on MacOS + $(CARGO) install -f --bins --path src/proxy --root=/usr/local/ # will fail on MacOS + +lint: githook + $(CARGO) clippy + cd src/core && $(CARGO) clippy --features="testmock" + cd src/core && $(CARGO) clippy --no-default-features + cd src/core && $(CARGO) clippy --no-default-features --features="zfs" + cd src/core && $(CARGO) clippy --no-default-features --features="cow" + cd src/core && $(CARGO) clippy --no-default-features --features="nft" + cd src/core && $(CARGO) clippy --no-default-features --features="cow nft" + cd src/server && $(CARGO) clippy --features="testmock" + cd src/server && $(CARGO) clippy --no-default-features + cd src/server && $(CARGO) clippy --no-default-features --features="zfs" + cd src/server && $(CARGO) clippy --no-default-features --features="cow" + cd src/server && $(CARGO) clippy --no-default-features --features="nft" + cd src/server && $(CARGO) clippy --no-default-features --features="cow nft" + cd src/proxy && $(CARGO) clippy --features="testmock" + +test: test_debug test_release + +test_debug: stop + $(CARGO) test -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/server && $(CARGO) test --features="testmock" -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/server && $(CARGO) test --features="testmock, cow" -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/server && $(CARGO) test --no-default-features --features="testmock" -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/server && $(CARGO) test --no-default-features --features="testmock, cow" -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/proxy && $(CARGO) test --features="testmock" -- --test-threads=1 --nocapture + -@ pkill -9 integration + +test_release: stop + $(CARGO) test --release -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/server && $(CARGO) test --release --features="testmock" -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/server && $(CARGO) test --release --features="testmock, cow" -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/server && $(CARGO) test --release --no-default-features --features="testmock" -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/server && $(CARGO) test --release --no-default-features --features="testmock, cow" -- --test-threads=1 --nocapture + -@ pkill -9 integration + cd src/proxy && $(CARGO) test --release --features="testmock" -- --test-threads=1 --nocapture + -@ pkill -9 integration -## Run all tests -test: - $(CARGO) test $(CARGO_FLAG) - -## Run clippy linter (treat warnings as errors) -lint: - $(CARGO) clippy $(CARGO_FLAG) -- -D warnings - -## Format code fmt: - $(CARGO) fmt - -## Check formatting without modifying -fmt-check: - $(CARGO) fmt -- --check + @ ./tools/fmt.sh -## Generate API documentation doc: - $(CARGO) doc --no-deps --document-private-items $(CARGO_FLAG) + $(CARGO) doc --open -p tt + $(CARGO) doc --open -p ttrexec + $(CARGO) doc --open -p ttproxy + $(CARGO) doc --open -p ttserver # will fail on MacOS + $(CARGO) doc --open -p ttcore # will fail on MacOS + +githook: + @mkdir -p ./.git/hooks # play with online gitlab-ci + @cp ./tools/githooks/pre-commit ./.git/hooks/ + +stop: + -@ pkill -9 ttproxy + -@ pkill -9 ttserver + -@ pkill -9 ttrexec-daemon -## Remove build artifacts clean: - $(CARGO) clean - -## Install release binaries to PREFIX/bin (no systemd setup) -install: release - @mkdir -p $(PREFIX)/bin - install -m 755 target/release/tt $(PREFIX)/bin/tt - install -m 755 target/release/tt-ctl $(PREFIX)/bin/tt-ctl - install -m 755 target/release/tt-agent $(PREFIX)/bin/tt-agent - @echo "Installed to $(PREFIX)/bin/" - -## Remove installed binaries -uninstall: - rm -f $(PREFIX)/bin/tt $(PREFIX)/bin/tt-ctl $(PREFIX)/bin/tt-agent - -## Deploy tt-agent on this host (requires root, idempotent) -deploy-agent: release - sudo target/release/tt deploy agent - -## Deploy tt-ctl (controller + web UI) on this host (requires root, idempotent) -deploy-ctl: release - sudo target/release/tt deploy ctl - -## Deploy both agent and controller on this host -deploy: release - sudo target/release/tt deploy all - -## Distributed deploy to all hosts in deploy.toml (via SSH) -deploy-dist: release - target/release/tt deploy dist deploy.toml - -## Show available targets -help: - @echo "TTstack build targets:" - @echo "" - @echo " Development:" - @echo " make build Debug build" - @echo " make release Optimized release build" - @echo " make test Run all tests" - @echo " make lint Run clippy" - @echo " make fmt Format code" - @echo " make doc Generate docs" - @echo " make clean Remove build artifacts" - @echo "" - @echo " Installation:" - @echo " make install Copy binaries to $(PREFIX)/bin/" - @echo " make uninstall Remove installed binaries" - @echo "" - @echo " Deployment (local, requires root, idempotent):" - @echo " make deploy-agent Deploy tt-agent on this host" - @echo " make deploy-ctl Deploy tt-ctl + web UI on this host" - @echo " make deploy Deploy both on this host" - @echo "" - @echo " Deployment (distributed, via SSH):" - @echo " make deploy-dist Deploy to fleet (reads deploy.toml)" - @echo "" - @echo " Images (auto-generate guest images):" - @echo " tt image recipes List available image recipes" - @echo " tt image create [--image-dir DIR]" - @echo " tt image create all --engine docker" + @ git clean -fdx + @ $(CARGO) clean + @ find . -type f -name "Cargo.lock" | xargs rm -f + +cleanall: clean + @ rm -rf client core core_def server server_def proxy rexec + @ find . -type d -name "target" | xargs rm -rf diff --git a/README.md b/README.md index abd88d2..29b4567 100644 --- a/README.md +++ b/README.md @@ -1,170 +1,198 @@ -# TTstack — Lightweight Private Cloud - -[![CI](https://github.com/rust-util-collections/TTstack/actions/workflows/ci.yml/badge.svg)](https://github.com/rust-util-collections/TTstack/actions/workflows/ci.yml) -[![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![Rust](https://img.shields.io/badge/rust-1.86%2B-orange.svg)](https://www.rust-lang.org) -[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20freebsd-green.svg)](#platform-support) - -TTstack is a lightweight private cloud platform for mid-size teams and -individual developers. Centralized management of VMs and containers -across multiple physical hosts. +> **版权声明:** +> +> 本项目最初以 "MIT" 开源证书创建于 GitEE 平台, 后续所有代码的更新, 均是首先发布到该仓库, 不存在任何商用版权问题, 欢迎所有感兴趣的公司试用. + +# tt + +[![pipeline status](https://git.lug.ustc.edu.cn/kt10/ttstack/badges/master/pipeline.svg)](https://git.lug.ustc.edu.cn/kt10/ttstack/-/commits/master) +[![coverage report](https://git.lug.ustc.edu.cn/kt10/ttstack/badges/master/coverage.svg)](https://git.lug.ustc.edu.cn/kt10/ttstack/-/commits/master) + +A light-weight 'private cloud solution' for SMEs, it can bring huge help and commercial value to start-up companies. + +面向中小型企业的轻量级私有云平台, 可快速生成各种虚拟机环境, 为产品兼容性验证和自动化测试等场景提供高效的基础环境. + +> #### 中小企业的效率悖论 +> +> 初创的中小企业, 技术实力薄弱, 很多生产手段都停留在刀耕火种的蛮荒时代. +> +> **这就形成了一个悖论:** +> +> > 原理上来讲, 初创公司要赶超大型公司, 必须要赢在效率; 确实, 在"人"的主观效率上, 如"管理流程"等方面, 大多数初创公司因为业务场景简单, 都能做到这一点；但"技术流程"上, 却是落后地一塌糊涂, 其结果当然是惨不忍睹, 尤如二战中的波兰骑兵, 高扬着马刀(原始工具)冲向德国人的坦克(现代工具), 不管马背上的骑士如何迅捷("人"的效率高), 都干不过坐在坦克("工具"的效率高)中的德国兵. +> > +> > 同时, 由于初创公司资金短缺, 很少有第三方的公司愿意去开拓这一块市场(无利可图); 而初创公司本身, 又没有足够的资源去自己解决, 这样就进入一个恶性循环, 永远处在"头痛医头, 脚痛医脚"的低效状态 , 直到公司倒闭, 或出现某个牛人以一己之力改变现状. +> +> 本项目以简洁易用为宗旨, 志在解决这个"无人问津"的悖论: +> +> - 专门面向中小企业设计, 分布式架构, 可扩展, 可伸缩 +> - 充分利用硬件资源: 通过云平台统一调度所有硬件资源, 大幅提升资源利用率和灵活性 +> - 极低的系统架设和维护成本: 运维人员通常只需半小时即可搭建起一套完整的 TT 私有云平台 +> - 极低的学习和使用成本: 终端用户通常只需十分钟即可熟练使用 TT 客户端创建需要的虚拟环境 +> - **省钱, 是的! 很省钱!** 你无需耗费巨资养活一个专门的云团队(OpenStack/K8S 专业人员的身价通常都很高) +> - 公有云真的很便宜? 很便利? 很安全? 用过的都知道答案 +> - ... + +## 主要用途 + +1. 广泛的平台兼容性验证 + - 可在如下两个方向上做任意的交叉组合 + 1. Linux、BSD、Windows、MacOS 等各种 OS 类别与版本 + 2. AMD64、X86、AArch64、ARM、MIPS、RISC-V、SPARC 等各种硬件平台 +2. 与 DevOps 系统配合, 实现自动化的 CI\CD 功能 +3. 用作原生编译平台 + - 直接申请全量的原生 OS 环境, 避免交叉编译的复杂度和潜在问题 +4. 用作短期或长期的调试环境 + - 可将 TT 视为传统的云平台, 申请虚拟机用于开发和测试 +5. 其它... + +## 技术特性 + +- 整洁高效的资源管理 + - 每个 VM 存在于独立的 Cgroup 中, 资源清理准确无误 + - [可选] 使用 FireCracker 快速创建大量的轻量级 MicroVM + - [默认] 使用 zfs 的 `snapshot\clone` 机制使 VM 获得原生 IO 性能 + - [默认] 使用 nftables 的 `SET\MAP` 等高级数据结构管理网络端口 + - 服务进程运行在单独的 `PID NS` 中, 服务退出会自动销毁所有资源 + - 通过 `Rust Drop` 机制自动管理 VM 生命周期 + - ... +- 分布式可扩展架构 + - 后端支持多机分布式架构, 对用户完全透明 +- 轻量级的通信模型 + - C\S 两端基于 UDP\SCTP 进行通信 + - 自研的远程命令执行工具, 效率远超 SSH 协议 +- 镜像源与服务解耦 + - 可随时增加受支持的系统镜像, 服务端不需要停机 + - 支持多种虚拟机引擎, 如: Qemu\FireCracker\Bhyve 等 + - 以镜像名称前缀识别虚拟机类型, 如: fire:centos-7.3:3.10.e17.x86_64 +- 使用`Rust`语言开发 + - 安全稳定 + - 高效运行 + - 文档齐备 + - 原生跨平台 + - ... ## Quick Start -```bash -make release -sudo tt deploy all # deploy agent + controller -sudo tt image create all --engine docker # generate Docker images -sudo tt image create alpine-cloud # generate QEMU cloud image (SSH-ready) - -tt config :9200 --api-key # key printed by deploy -tt host add :9100 # register a host +#### 编译 -tt env create demo --image alpine-cloud --engine qemu \ - --ssh-key ~/.ssh/id_ed25519.pub -tt env show demo # see port mappings -ssh root@ -p # key-based auth +```shell +make install +export PATH=~/.cargo/bin:$PATH ``` -## VM Access - -| Engine | How to access | -|--------|--------------| -| **QEMU** cloud images | `ssh root@ -p ` (SSH key injected via cloud-init) | -| **QEMU** custom images | SSH via port forwarding (your own key setup) | -| **Docker** | SSH (if sshd in image) or `docker exec` from host | -| **Firecracker** | serial console only | -| **Bhyve** (FreeBSD) | SSH via port forwarding | - -QEMU cloud images auto-configure via **cloud-init**: SSH public keys, -networking — all set on first boot. See [docs/guest-images.md](docs/guest-images.md). - -## Security - -All `/api/*` endpoints require a Bearer token when `--api-key` is set -(auto-generated on deploy). The web dashboard (`/`) remains open. - -```bash -# Set in deploy.toml: -[general] -api_key = "your-secret-key" - -# Or configure CLI directly: -tt config --api-key - -# Or via environment: -export TT_API_KEY=your-secret-key +#### 启动服务端 + +> **注意** +> +> 镜像文件一定**不**能存放在'/tmp'或其子目录下, 会导致无法扫描到镜像信息(ttserver 的'/tmp'路径私有的, 与外界环境互相隔离). + +```shell +# Slave Server 1 +ttserver \ + --image-path /home/images \ + --cpu-total 2 \ + --mem-total $[4 * 1024 * 1024] \ + --disk-total $[40 * 1024 * 1024] \ + --serv-addr 127.0.0.1 \ + --serv-port 20000 + +# Slave Server 2 +ttserver \ + --image-path /home/images \ + --cpu-total 2 \ + --mem-total $[4 * 1024 * 1024] \ + --disk-total $[40 * 1024 * 1024] \ + --serv-addr 127.0.0.1 \ + --serv-port 20001 + +# Proxy, 分布式代理服务, 负责调度各 Slave Server 的资源 +ttproxy \ + --proxy-addr 127.0.0.1:20002 \ + --server-set 127.0.0.1:20000,127.0.0.1:20001 ``` -## Built-in Images +#### 客户端操作 -12 ready-to-use recipes — deploy and start creating VMs immediately: +> **Tips** +> - 完整的客户端操作文档, 参见: [《用户指南》](./documents/user_guide.md) +> - "/home/images" 路径下需要存在可正常启动的 Qemu 镜像文件 +> - 镜像文件中的 "/etc/rc.local" 文件需要替换为本项目定制的 "[rc.local](./tools/images/linux_vm/rc.local)" -| Recipe | Engine | Description | -|--------|--------|-------------| -| `alpine` `debian` `ubuntu` `rockylinux` | Docker | Base OS containers | -| `nginx` `redis` `postgres` | Docker | Popular services | -| `fc-alpine` | Firecracker | Alpine microVM (~50MB) | -| `alpine-cloud` `debian-cloud` `ubuntu-cloud` | QEMU | SSH-ready cloud images | -| `freebsd-base` | Jail | FreeBSD 14.3 base | +```shell +# 配置服务端地址, +# 既可以是 Proxy 的地址, +# 也可以是各个独立的 Slave Server 地址, +# 这里配置成 Proxy 的地址, 以演示分布式架构的调度效果 +tt config --serv-addr 127.0.0.1 --serv-port 20002 -```bash -tt image recipes # list all -sudo tt image create all --engine docker # all Docker images -sudo tt image create alpine-cloud # one QEMU cloud image -sudo tt image create all # everything for this platform -``` +# 查看客户端本地信息 +tt status -See [docs/guest-images.md](docs/guest-images.md) for custom image creation. +# 查看服务端资源信息 +tt status --server -## Key Features +# 创建一个 "ENV", +# TT 中的基本管理单位为 ENV (一组 VM 的集合), +# 创建的 VM 类别是以系统前缀匹配的, 不区分大小写, +# 如: +# - cent 会匹配到所有 CentOS 系统 +# - ubuntu2004 只会匹配到 Ubuntu2004 一个系统 +tt env add TEST --os-prefix=cent,ubuntu2004 -- **Multi-engine**: QEMU/KVM, Firecracker, Docker/Podman (Linux); Bhyve, Jail (FreeBSD) -- **Multi-host fleet**: up to 50 hosts, 1000 VM instances, best-fit scheduling -- **Environments**: group VMs with lifecycle control and auto-expiry (default 6h) -- **Storage backends**: ZFS zvol (instant clone), plain qcow2 file copies -- **SSH key injection**: provide public keys at create time; port 22 auto-included -- **Web dashboard**: built-in monitoring UI at `http://:9200` -- **Simple deploy**: three binaries, SQLite, one command (`tt deploy all`) +# 查看已创建的 ENV 列表 +tt env list -## Architecture +# 查看已创建的某个 ENV 的详情 +tt env show TEST -``` -┌──────────┐ ┌──────────────┐ ┌───────────┐ -│ tt CLI ├──── HTTP ──►│ tt-ctl ├──── HTTP ──►│ tt-agent │ × N -└──────────┘ │ (controller) │ │ (per-host)│ -┌──────────┐ │ + Web UI │ └─────┬─────┘ -│ Browser ├──── HTTP ──►└──────┬───────┘ │ -└──────────┘ │ VM engines + storage - SQLite DB -``` - -| Binary | Role | -|--------|------| -| **tt** | CLI client | -| **tt-ctl** | Central controller: scheduling, state, web UI | -| **tt-agent** | Host agent: VM lifecycle, images, networking | - -## CLI Reference - -``` -tt config [--api-key ] Set controller address and API key -tt status Fleet-wide status - -tt host add/list/show/remove Manage hosts -tt env create/list/show/delete Manage environments -tt env stop/start Lifecycle control +# 在 ENV 的所有 VM 上执行相同的命令 +tt env run TEST --use-ssh --cmd 'ls /' -tt image list/recipes/create Manage images -tt deploy agent/ctl/all/dist Deploy TTstack +# 删除 ENV, +# 其中所有的 VM 及其相关数据都会被清理 +tt env del TEST ``` -### `env create` options - -| Option | Description | Default | -|--------|-------------|---------| -| `-i, --image ` | Base image (repeatable) | *required* | -| `--engine ` | qemu, firecracker, docker, bhyve, jail | qemu | -| `--cpu ` | vCPUs per VM | 2 | -| `--mem ` | Memory per VM | 1024 | -| `--disk ` | Disk per VM | 40960 | -| `--dup ` | Replicas per image | 1 | -| `--ssh-key ` | SSH public key file (repeatable) | *required for VMs* | -| `-p, --port ` | Guest port to expose (repeatable) | — | -| `--lifetime ` | Auto-expiry (0 = 6h default) | 21600 | -| `--deny-outgoing` | Block outbound traffic | false | +## 详细文档 -## Platform Support +- [开发路线(RoadMap)](./documents/roadmap.md) +- [终端用户指南](./documents/user_guide.md) +- [系统管理指南](./documents/system_admin.md) +- [架构设计与技术选型](./documents/arch_design.md) +- [项目结构与代码规模](./documents/code_about.md) -| Platform | Engines | Networking | -|----------|---------|------------| -| **Linux** | QEMU/KVM, Firecracker, Docker/Podman | nftables NAT | -| **FreeBSD** | Bhyve, Jail | PF NAT | +> #### 接口文档 +> +> ```shell +> # 在 Rust 开发环境下执行 +> make doc +> ``` -## Documentation +## BUG -| Document | Contents | -|----------|----------| -| [docs/deployment.md](docs/deployment.md) | Full deployment guide, config reference, directory layout | -| [docs/guest-images.md](docs/guest-images.md) | Image formats, custom image creation, VM access details | -| [docs/rest-api.md](docs/rest-api.md) | REST API endpoints with curl examples | -| [docs/compatibility.md](docs/compatibility.md) | Platform test results and known issues | +- 单个 ENV 超过 400+ VM 时, 可能出现异常 + - 原因是太大的 ENV 会超过 UDP 单次通信的最大载荷 + - 目前采用了数据压缩的方式予以缓解, 后续将改用 HTTP\SCTP -## Project Structure +## Statistics ``` -TTstack/ -├── Cargo.toml Workspace -├── Makefile Build + deploy targets -├── tools/ -│ └── deploy.toml.example Fleet configuration template -└── crates/ - ├── core/ Shared library (engines, storage, networking, models) - ├── agent/ Host agent (tt-agent) - ├── ctl/ Controller (tt-ctl) - └── cli/ CLI client (tt) +(git)-[master]-% tokei +=============================================================================== + Language Files Lines Code Comments Blanks +=============================================================================== + BASH 1 5 2 1 2 + Makefile 1 108 92 0 16 + Shell 8 278 198 30 50 + TOML 10 234 192 1 41 +------------------------------------------------------------------------------- + Markdown 10 662 0 492 170 + |- Shell 4 372 333 23 16 + (Total) 1034 333 515 186 +------------------------------------------------------------------------------- + Rust 77 9182 7770 287 1125 + |- Markdown 71 803 41 707 55 + (Total) 9985 7811 994 1180 +=============================================================================== + Total 107 11644 8628 1541 1475 +=============================================================================== ``` - -## License - -MIT diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml deleted file mode 100644 index 6c1ab6e..0000000 --- a/crates/agent/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "tt-agent" -description = "TTstack host agent — manages VMs and containers on a single physical host." -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true - -[[bin]] -name = "tt-agent" -path = "src/main.rs" - -[dependencies] -ttcore = { path = "../core" } -serde = { workspace = true } -serde_json = { workspace = true } -ruc = { workspace = true } -rusqlite = { workspace = true } -tokio = { workspace = true } -axum = { workspace = true } -clap = { workspace = true } -uuid = { workspace = true } - -[dev-dependencies] -tempfile = { workspace = true } diff --git a/crates/agent/src/auth.rs b/crates/agent/src/auth.rs deleted file mode 100644 index b4eb94e..0000000 --- a/crates/agent/src/auth.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! API key authentication middleware. - -use axum::body::Body; -use axum::http::{Request, StatusCode}; -use axum::middleware::Next; -use axum::response::{IntoResponse, Response}; - -/// Create an auth middleware function that validates Bearer tokens. -/// -/// Returns a closure suitable for `axum::middleware::from_fn`. -pub fn make_auth_layer( - expected_key: String, -) -> impl Fn( - Request, - Next, -) -> std::pin::Pin + Send>> -+ Clone -+ Send -+ Sync -+ 'static { - move |req: Request, next: Next| { - let expected = expected_key.clone(); - Box::pin(async move { - let auth_header = req - .headers() - .get("authorization") - .and_then(|v| v.to_str().ok()); - - match auth_header { - Some(value) - if value - .strip_prefix("Bearer ") - .is_some_and(|t| ttcore::auth::constant_time_eq(t, &expected)) => - { - next.run(req).await - } - _ => ( - StatusCode::UNAUTHORIZED, - axum::Json(ttcore::api::ApiResp::<()>::err( - "invalid or missing API key", - )), - ) - .into_response(), - } - }) - } -} diff --git a/crates/agent/src/config.rs b/crates/agent/src/config.rs deleted file mode 100644 index d5892d0..0000000 --- a/crates/agent/src/config.rs +++ /dev/null @@ -1,194 +0,0 @@ -//! Agent configuration. - -use clap::Parser; -use ttcore::model::Storage; - -/// TTstack host agent — manages VMs and containers on this host. -#[derive(Parser, Debug)] -#[command(name = "tt-agent", version)] -pub struct Config { - /// Listen address for the HTTP API. - #[arg(long, default_value = "0.0.0.0:9100")] - pub listen: String, - - /// Directory containing base VM/container images. - #[arg(long, default_value = "/home/ttstack/images")] - pub image_dir: String, - - /// Directory for runtime VM image clones. - #[arg(long, default_value = "/home/ttstack/runtime")] - pub runtime_dir: String, - - /// Directory for persistent state (SQLite database). - #[arg(long, default_value = "/home/ttstack/data")] - pub data_dir: String, - - /// Storage backend: file or zvol. - #[arg(long, default_value = "file")] - pub storage: String, - - /// Total CPU cores available for VMs (0 = auto-detect). - #[arg(long, default_value_t = 0)] - pub cpu_total: u32, - - /// Total memory for VMs in MiB (0 = auto-detect). - #[arg(long, default_value_t = 0)] - pub mem_total: u32, - - /// Total disk for VMs in MiB (default: ~200 GiB). - #[arg(long, default_value_t = 200 * 1024)] - pub disk_total: u32, - - /// Unique host identifier (auto-generated if not set). - #[arg(long)] - pub host_id: Option, - /// API key for authentication. If set, all API requests must include - /// `Authorization: Bearer `. Can also be provided via TT_API_KEY env var. - #[arg(long, env = "TT_API_KEY")] - pub api_key: Option, -} - -impl Config { - pub fn storage_kind(&self) -> Storage { - self.storage.parse().unwrap_or(Storage::File) - } - - /// Auto-detect CPU count if set to 0. - pub fn effective_cpu(&self) -> u32 { - if self.cpu_total == 0 { - std::thread::available_parallelism() - .map(|n| n.get() as u32) - .unwrap_or(4) - } else { - self.cpu_total - } - } - - /// Auto-detect memory if set to 0 (read from /proc/meminfo). - pub fn effective_mem(&self) -> u32 { - if self.mem_total == 0 { - read_total_mem_mb().unwrap_or(8192) - } else { - self.mem_total - } - } -} - -/// Read total system memory in MB. -/// -/// Uses `/proc/meminfo` on Linux, `sysctl hw.physmem` on FreeBSD, -/// `sysctl hw.memsize` on macOS. -fn read_total_mem_mb() -> Option { - #[cfg(target_os = "linux")] - { - let content = std::fs::read_to_string("/proc/meminfo").ok()?; - parse_mem_total(&content) - } - #[cfg(target_os = "freebsd")] - { - let output = std::process::Command::new("sysctl") - .arg("-n") - .arg("hw.physmem") - .output() - .ok()?; - let bytes: u64 = String::from_utf8_lossy(&output.stdout) - .trim() - .parse() - .ok()?; - return Some((bytes / (1024 * 1024)) as u32); - } - #[cfg(not(any(target_os = "linux", target_os = "freebsd")))] - { - let output = std::process::Command::new("sysctl") - .arg("-n") - .arg("hw.memsize") - .output() - .ok()?; - let bytes: u64 = String::from_utf8_lossy(&output.stdout) - .trim() - .parse() - .ok()?; - Some((bytes / (1024 * 1024)) as u32) - } -} - -/// Parse MemTotal from /proc/meminfo content. -#[cfg(any(target_os = "linux", test))] -fn parse_mem_total(content: &str) -> Option { - for line in content.lines() { - if line.starts_with("MemTotal:") { - let kb: u64 = line.split_whitespace().nth(1)?.parse().ok()?; - return Some((kb / 1024) as u32); - } - } - None -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_meminfo() { - let content = "\ -MemTotal: 131696312 kB -MemFree: 1234567 kB -MemAvailable: 9876543 kB"; - assert_eq!(parse_mem_total(content), Some(131696312 / 1024)); - } - - #[test] - fn parse_meminfo_missing() { - assert_eq!(parse_mem_total("nothing here"), None); - } - - #[test] - fn parse_meminfo_malformed() { - assert_eq!(parse_mem_total("MemTotal: notanumber kB"), None); - } - - #[test] - fn storage_kind_default() { - // Default is "file" - let cfg = Config::parse_from(["tt-agent"]); - assert_eq!(cfg.storage_kind(), Storage::File); - } - - #[test] - fn storage_kind_zvol() { - let cfg = Config::parse_from(["tt-agent", "--storage", "zvol"]); - assert_eq!(cfg.storage_kind(), Storage::Zvol); - } - - #[test] - fn storage_kind_invalid_falls_back() { - let cfg = Config::parse_from(["tt-agent", "--storage", "foo"]); - assert_eq!(cfg.storage_kind(), Storage::File); - } - - #[test] - fn effective_cpu_explicit() { - let cfg = Config::parse_from(["tt-agent", "--cpu-total", "16"]); - assert_eq!(cfg.effective_cpu(), 16); - } - - #[test] - fn effective_cpu_auto() { - let cfg = Config::parse_from(["tt-agent"]); - // Auto-detect: should be > 0 - assert!(cfg.effective_cpu() > 0); - } - - #[test] - fn effective_mem_explicit() { - let cfg = Config::parse_from(["tt-agent", "--mem-total", "4096"]); - assert_eq!(cfg.effective_mem(), 4096); - } - - #[test] - fn effective_mem_auto() { - let cfg = Config::parse_from(["tt-agent"]); - // On any real machine, should detect > 0 - assert!(cfg.effective_mem() > 0); - } -} diff --git a/crates/agent/src/handler.rs b/crates/agent/src/handler.rs deleted file mode 100644 index 4e69d0e..0000000 --- a/crates/agent/src/handler.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! HTTP API handlers for the host agent. - -use crate::runtime::Runtime; -use axum::Json; -use axum::extract::{Path, State}; -use axum::http::StatusCode; -use axum::response::IntoResponse; -use std::sync::{Arc, Mutex, MutexGuard}; -use ttcore::api::*; -use ttcore::model::Vm; - -/// Shared application state. -pub type AppState = Arc>; - -/// Lock the runtime mutex, recovering from poisoning if a prior -/// handler panicked while holding the lock. -fn lock_rt(rt: &AppState) -> MutexGuard<'_, Runtime> { - rt.lock().unwrap_or_else(|e| { - eprintln!("[agent] WARN: runtime mutex was poisoned, recovering"); - e.into_inner() - }) -} - -/// GET /api/info — report host resources and capabilities. -pub async fn get_info(State(rt): State) -> impl IntoResponse { - let rt = lock_rt(&rt); - Json(ApiResp::success(rt.agent_info())) -} - -/// GET /api/images — list available base images. -pub async fn list_images(State(rt): State) -> impl IntoResponse { - let rt = lock_rt(&rt); - Json(ApiResp::success(rt.list_images())) -} - -/// POST /api/vms — create a new VM. -pub async fn create_vm( - State(rt): State, - Json(req): Json, -) -> impl IntoResponse { - let mut rt = lock_rt(&rt); - match rt.create_vm(&req) { - Ok(vm) => ( - StatusCode::CREATED, - Json(ApiResp::success(CreateVmResp { vm })), - ), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiResp::::err(e.to_string())), - ), - } -} - -/// GET /api/vms — list all VMs on this host. -pub async fn list_vms(State(rt): State) -> impl IntoResponse { - let rt = lock_rt(&rt); - Json(ApiResp::success(rt.list_vms())) -} - -/// GET /api/vms/:id — get a specific VM. -pub async fn get_vm(State(rt): State, Path(id): Path) -> impl IntoResponse { - let rt = lock_rt(&rt); - match rt.get_vm(&id) { - Some(vm) => (StatusCode::OK, Json(ApiResp::success(vm))), - None => ( - StatusCode::NOT_FOUND, - Json(ApiResp::::err(format!("VM not found: {id}"))), - ), - } -} - -/// DELETE /api/vms/:id — destroy a VM. -pub async fn destroy_vm(State(rt): State, Path(id): Path) -> impl IntoResponse { - let mut rt = lock_rt(&rt); - match rt.destroy_vm(&id) { - Ok(()) => (StatusCode::OK, Json(ApiRespEmpty::ok())), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiRespEmpty::err(e.to_string())), - ), - } -} - -/// POST /api/vms/:id/stop — stop a VM. -pub async fn stop_vm(State(rt): State, Path(id): Path) -> impl IntoResponse { - let mut rt = lock_rt(&rt); - match rt.stop_vm(&id) { - Ok(()) => (StatusCode::OK, Json(ApiRespEmpty::ok())), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiRespEmpty::err(e.to_string())), - ), - } -} - -/// POST /api/vms/:id/start — start a stopped VM. -pub async fn start_vm(State(rt): State, Path(id): Path) -> impl IntoResponse { - let mut rt = lock_rt(&rt); - match rt.start_vm(&id) { - Ok(()) => (StatusCode::OK, Json(ApiRespEmpty::ok())), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiRespEmpty::err(e.to_string())), - ), - } -} diff --git a/crates/agent/src/main.rs b/crates/agent/src/main.rs deleted file mode 100644 index 824d74f..0000000 --- a/crates/agent/src/main.rs +++ /dev/null @@ -1,102 +0,0 @@ -//! TTstack host agent entry point. -//! -//! The agent runs on each physical host, managing local VMs/containers -//! and exposing an HTTP API for the central controller. -//! -//! Supported platforms: -//! - **Linux**: all engines (Qemu, Firecracker, Docker) -//! - **FreeBSD**: Bhyve, Jail -//! - **Other Unix** (macOS, etc.): Docker/Podman only - -mod auth; -mod config; -mod handler; -mod runtime; - -use axum::Router; -use axum::routing::{get, post}; -use clap::Parser; -use config::Config; -use handler::AppState; -use runtime::Runtime; -use std::sync::{Arc, Mutex}; -use ttcore::model::Resource; - -#[tokio::main] -async fn main() { - let cfg = Config::parse(); - - let db_path = format!("{}/agent.db", cfg.data_dir); - std::fs::create_dir_all(&cfg.data_dir).unwrap_or_else(|e| { - eprintln!("Failed to create data dir {}: {e}", cfg.data_dir); - std::process::exit(1); - }); - - let host_id = runtime::resolve_host_id(&db_path, cfg.host_id.clone()).unwrap_or_else(|e| { - eprintln!("Failed to resolve host_id: {e}"); - std::process::exit(1); - }); - - let resource = Resource { - cpu_total: cfg.effective_cpu(), - mem_total: cfg.effective_mem(), - disk_total: cfg.disk_total, - ..Default::default() - }; - - let rt = Runtime::new( - host_id.clone(), - cfg.storage_kind(), - cfg.image_dir.clone(), - cfg.runtime_dir.clone(), - &db_path, - resource, - ) - .unwrap_or_else(|e| { - eprintln!("Failed to initialize runtime: {e}"); - std::process::exit(1); - }); - - let state: AppState = Arc::new(Mutex::new(rt)); - - let app = Router::new() - .route("/api/info", get(handler::get_info)) - .route("/api/images", get(handler::list_images)) - .route("/api/vms", get(handler::list_vms).post(handler::create_vm)) - .route( - "/api/vms/{id}", - get(handler::get_vm).delete(handler::destroy_vm), - ) - .route("/api/vms/{id}/stop", post(handler::stop_vm)) - .route("/api/vms/{id}/start", post(handler::start_vm)) - .with_state(state); - - let app = if let Some(key) = cfg.api_key { - eprintln!("API key authentication enabled"); - app.layer(axum::middleware::from_fn(auth::make_auth_layer(key))) - } else { - eprintln!("WARNING: no --api-key set, all agent endpoints are unauthenticated!"); - app - }; - - let listener = tokio::net::TcpListener::bind(&cfg.listen) - .await - .unwrap_or_else(|e| { - eprintln!("Failed to bind {}: {e}", cfg.listen); - std::process::exit(1); - }); - - eprintln!("tt-agent [{host_id}] listening on {}", cfg.listen); - - axum::serve(listener, app) - .with_graceful_shutdown(shutdown_signal()) - .await - .unwrap_or_else(|e| eprintln!("Server error: {e}")); - - eprintln!("tt-agent shutting down"); -} - -async fn shutdown_signal() { - let _ = tokio::signal::ctrl_c().await; - eprintln!("received shutdown signal"); -} diff --git a/crates/agent/src/runtime.rs b/crates/agent/src/runtime.rs deleted file mode 100644 index 3a036c6..0000000 --- a/crates/agent/src/runtime.rs +++ /dev/null @@ -1,780 +0,0 @@ -//! VM runtime management on a single host. -//! -//! Owns the lifecycle of all VMs on this host, backed by SQLite for -//! crash-recoverable persistent state. - -use ruc::*; -use rusqlite::Connection; -use std::collections::{BTreeMap, HashSet}; -use std::sync::atomic::{AtomicU32, Ordering}; -use ttcore::api::{AgentInfo, CreateVmReq}; -use ttcore::engine; -use ttcore::model::*; -use ttcore::net; -use ttcore::storage::{self, ImageStore}; - -/// Manages all VMs on this host. -pub struct Runtime { - pub host_id: String, - db: Connection, - engines: Vec, - store: Box, - storage: Storage, - image_dir: String, - runtime_dir: String, - pub resource: Resource, - next_ip_idx: AtomicU32, -} - -impl Runtime { - /// Initialize the runtime, restoring state from SQLite if available. - pub fn new( - host_id: String, - storage: Storage, - image_dir: String, - runtime_dir: String, - db_path: &str, - resource: Resource, - ) -> Result { - std::fs::create_dir_all(&image_dir).c(d!("create image_dir"))?; - std::fs::create_dir_all(&runtime_dir).c(d!("create runtime_dir"))?; - std::fs::create_dir_all(ttcore::model::RUN_DIR).c(d!("create run dir"))?; - - let db = Connection::open(db_path).c(d!("open agent db"))?; - init_db(&db)?; - - let store = storage::create_store(storage); - let engines = detect_engines(); - - // Restore state from database - let vms = load_all_vms(&db)?; - - // Clean up VMs stuck in "Creating" state (agent crashed mid-creation). - // These VMs may have partial resources allocated that need cleanup. - for vm in &vms { - if vm.state == VmState::Creating { - eprintln!( - "[agent] cleaning up orphaned VM {} (stuck in Creating state)", - vm.id - ); - let eng = engine::create_engine(vm.engine); - let _ = eng.destroy(vm); - let _ = storage::create_store(storage) - .remove_image(&format!("{}/clone-{}", runtime_dir, vm.id)); - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - net::destroy_tap(&vm.id).unwrap_or(()); - let _ = delete_vm(&db, &vm.id); - } - } - - // Reload after cleanup - let vms = load_all_vms(&db)?; - - let max_idx = vms - .iter() - .filter_map(|vm| ip_to_index(&vm.ip)) - .max() - .unwrap_or(0); - - let mut cpu_used = 0u32; - let mut mem_used = 0u32; - let mut disk_used = 0u32; - let mut vm_count = 0u32; - for vm in &vms { - if vm.state == VmState::Running || vm.state == VmState::Paused { - cpu_used += vm.cpu; - mem_used += vm.mem; - disk_used += vm.disk; - vm_count += 1; - } - } - - let resource = Resource { - cpu_used, - mem_used, - disk_used, - vm_count, - ..resource - }; - - // Set up networking (only on platforms with host-managed networking) - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - { - net::setup_bridge().c(d!("bridge setup"))?; - net::setup_nat().c(d!("NAT setup"))?; - } - - // Restore network rules for persisted VMs - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - for vm in &vms { - if vm.state == VmState::Running || vm.state == VmState::Paused { - if let Err(e) = net::create_tap(&vm.id, &vm.ip) { - eprintln!("[agent] WARN: failed to restore TAP for VM {}: {e}", vm.id); - } - for (&guest, &host) in &vm.port_map { - if let Err(e) = net::add_port_forward(host, &vm.ip, guest) { - eprintln!( - "[agent] WARN: failed to restore port forward {}->{}:{} for VM {}: {e}", - host, vm.ip, guest, vm.id - ); - } - } - } - } - - Ok(Self { - host_id, - db, - engines, - store, - storage, - image_dir, - runtime_dir, - resource, - next_ip_idx: AtomicU32::new(max_idx + 1), - }) - } - - /// Create a new VM. - pub fn create_vm(&mut self, req: &CreateVmReq) -> Result { - // Input validation - validate_name(&req.vm_id, "vm_id").map_err(|e| eg!(e))?; - validate_name(&req.image, "image").map_err(|e| eg!(e))?; - - if req.cpu == 0 { - return Err(eg!("cpu must be > 0")); - } - if req.mem == 0 { - return Err(eg!("mem must be > 0")); - } - if req.disk == 0 { - return Err(eg!("disk must be > 0")); - } - - if !self.resource.can_fit(req.cpu, req.mem, req.disk) { - return Err(eg!("insufficient resources on host {}", self.host_id)); - } - - if self.resource.vm_count as usize >= MAX_VMS { - return Err(eg!("VM limit reached")); - } - - // Allocate IP, skipping any indices already in use by existing VMs - let existing_ips: HashSet = load_all_vms(&self.db) - .unwrap_or_default() - .into_iter() - .map(|vm| vm.ip) - .collect(); - let (ip_idx, ip) = loop { - let idx = self.next_ip_idx.fetch_add(1, Ordering::SeqCst); - if idx > 65000 { - return Err(eg!("IP address space exhausted")); - } - let candidate = net::vm_ip(idx); - if !existing_ips.contains(&candidate) { - break (idx, candidate); - } - }; - - // Docker/Podman manages its own images, networking, and port mapping; - // other engines need local image clones, TAP devices, and nftables rules. - // Host-managed networking is only available on Linux and FreeBSD. - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - let host_managed_net = req.engine != Engine::Docker; - #[cfg(not(any(target_os = "linux", target_os = "freebsd")))] - let host_managed_net = false; - - let clone_path = format!("{}/clone-{}", self.runtime_dir, req.vm_id); - if host_managed_net { - let base_image = format!("{}/{}", self.image_dir, req.image); - self.store - .clone_image(&base_image, &clone_path) - .c(d!("image clone"))?; - } - - if host_managed_net { - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - net::create_tap(&req.vm_id, &ip).c(d!("TAP setup"))?; - } - - // Allocate port mappings (use checked arithmetic to avoid u16 overflow) - // Always include port 22 for SSH access - let mut ports = req.ports.clone(); - if !ports.contains(&22) { - ports.insert(0, 22); - } - let mut port_map = BTreeMap::new(); - let base_port = 20000u32.saturating_add(ip_idx.saturating_mul(100)); - for (i, &guest_port) in ports.iter().enumerate() { - let host_port = base_port.saturating_add(i as u32); - if host_port > 65535 { - break; // silently skip ports that can't be allocated - } - port_map.insert(guest_port, host_port as u16); - } - - let mut vm = Vm { - id: req.vm_id.clone(), - env_id: req.env_id.clone(), - host_id: self.host_id.clone(), - image: req.image.clone(), - engine: req.engine, - cpu: req.cpu, - mem: req.mem, - disk: req.disk, - ip: ip.clone(), - port_map: port_map.clone(), - state: VmState::Creating, - created_at: now(), - }; - - save_vm(&self.db, &vm)?; - - // Resolve the disk path and format from the storage backend - let disk_path = self.store.resolve_disk(&clone_path); - let disk_format = self.store.disk_format(); - - // Launch using the appropriate engine - let eng = engine::create_engine(req.engine); - if let Err(e) = eng.create(&vm, &disk_path, disk_format, &req.ssh_keys) { - if host_managed_net { - let _ = self.store.remove_image(&clone_path); - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - net::destroy_tap(&req.vm_id).unwrap_or(()); - } - delete_vm(&self.db, &vm.id)?; - return Err(e).c(d!("engine create")); - } - - // Set up nftables port forwarding and outgoing rules. - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - if host_managed_net { - let post_create = || -> Result<()> { - for (&guest_port, &host_port) in &port_map { - net::add_port_forward(host_port, &ip, guest_port).c(d!("port forward"))?; - } - if req.deny_outgoing { - net::deny_outgoing(&ip).c(d!("deny outgoing"))?; - } - Ok(()) - }; - - if let Err(e) = post_create() { - let _ = net::remove_port_forwards(&ip); - let _ = net::allow_outgoing(&ip); - let _ = eng.destroy(&vm); - let _ = self.store.remove_image(&clone_path); - let _ = net::destroy_tap(&req.vm_id); - let _ = delete_vm(&self.db, &vm.id); - return Err(e).c(d!("post-create setup")); - } - } - - // Update resource tracking - self.resource.cpu_used += req.cpu; - self.resource.mem_used += req.mem; - self.resource.disk_used += req.disk; - self.resource.vm_count += 1; - - vm.state = VmState::Running; - save_vm(&self.db, &vm)?; - - Ok(vm) - } - - pub fn stop_vm(&mut self, vm_id: &str) -> Result<()> { - let mut vm = load_vm(&self.db, vm_id)?.ok_or_else(|| eg!("VM not found: {}", vm_id))?; - - match vm.state { - VmState::Running => {} - VmState::Paused | VmState::Stopped => return Ok(()), - _ => return Err(eg!("cannot stop VM in state {}", vm.state)), - } - - let eng = engine::create_engine(vm.engine); - eng.stop(&vm).c(d!("stop VM"))?; - - // FC/QEMU pause (preserve process); others fully stop - let pauses = matches!(vm.engine, Engine::Qemu | Engine::Firecracker); - if pauses { - vm.state = VmState::Paused; - } else { - vm.state = VmState::Stopped; - self.resource.cpu_used = self.resource.cpu_used.saturating_sub(vm.cpu); - self.resource.mem_used = self.resource.mem_used.saturating_sub(vm.mem); - } - save_vm(&self.db, &vm)?; - - Ok(()) - } - - pub fn start_vm(&mut self, vm_id: &str) -> Result<()> { - let mut vm = load_vm(&self.db, vm_id)?.ok_or_else(|| eg!("VM not found: {}", vm_id))?; - - let prev_state = vm.state; - match prev_state { - VmState::Stopped | VmState::Paused => {} - VmState::Running => return Ok(()), - _ => return Err(eg!("cannot start VM in state {}", vm.state)), - } - - // Only check/allocate resources when resuming from fully Stopped - if prev_state == VmState::Stopped && !self.resource.can_fit(vm.cpu, vm.mem, 0) { - return Err(eg!("insufficient resources to restart VM")); - } - - let eng = engine::create_engine(vm.engine); - eng.start(&vm).c(d!("start VM"))?; - - vm.state = VmState::Running; - save_vm(&self.db, &vm)?; - - if prev_state == VmState::Stopped { - self.resource.cpu_used += vm.cpu; - self.resource.mem_used += vm.mem; - } - - Ok(()) - } - - pub fn destroy_vm(&mut self, vm_id: &str) -> Result<()> { - let vm = match load_vm(&self.db, vm_id)? { - Some(vm) => vm, - None => return Ok(()), - }; - - let eng = engine::create_engine(vm.engine); - let _ = eng.destroy(&vm); - - // Clean up host-managed networking and image clones. - // Docker handles its own network teardown. - #[cfg(any(target_os = "linux", target_os = "freebsd"))] - if vm.engine != Engine::Docker { - net::remove_port_forwards(&vm.ip).unwrap_or(()); - net::allow_outgoing(&vm.ip).unwrap_or(()); - net::destroy_tap(vm_id).unwrap_or(()); - } - - if vm.engine != Engine::Docker { - let clone_path = format!("{}/clone-{}", self.runtime_dir, vm_id); - let _ = self.store.remove_image(&clone_path); - } - - if vm.state == VmState::Running - || vm.state == VmState::Paused - || vm.state == VmState::Creating - { - self.resource.cpu_used = self.resource.cpu_used.saturating_sub(vm.cpu); - self.resource.mem_used = self.resource.mem_used.saturating_sub(vm.mem); - } - self.resource.disk_used = self.resource.disk_used.saturating_sub(vm.disk); - self.resource.vm_count = self.resource.vm_count.saturating_sub(1); - - delete_vm(&self.db, vm_id)?; - - Ok(()) - } - - pub fn get_vm(&self, vm_id: &str) -> Option { - load_vm(&self.db, vm_id).ok().flatten() - } - - pub fn list_vms(&self) -> Vec { - load_all_vms(&self.db).unwrap_or_default() - } - - pub fn list_images(&self) -> Vec { - self.store.list_images(&self.image_dir).unwrap_or_default() - } - - pub fn agent_info(&self) -> AgentInfo { - AgentInfo { - host_id: self.host_id.clone(), - resource: self.resource.clone(), - engines: self.engines.clone(), - storage: self.storage, - images: self.list_images(), - } - } -} - -// ── SQLite Schema & Operations ────────────────────────────────────── - -/// Current agent schema version. -const SCHEMA_VERSION: u32 = 1; - -fn init_db(db: &Connection) -> Result<()> { - db.execute_batch( - "PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - CREATE TABLE IF NOT EXISTS _meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - );", - ) - .c(d!("init meta table"))?; - - let current = get_schema_version(db)?; - - if current > SCHEMA_VERSION { - return Err(eg!( - "agent DB schema v{} is newer than this binary (v{}); upgrade TTstack first", - current, - SCHEMA_VERSION - )); - } - - if current < 1 { - db.execute_batch( - "CREATE TABLE IF NOT EXISTS vms ( - id TEXT PRIMARY KEY, - data TEXT NOT NULL - );", - ) - .c(d!("migration v1"))?; - } - - // Future migrations: if current < 2 { ... } - - set_schema_version(db, SCHEMA_VERSION)?; - if current < SCHEMA_VERSION { - eprintln!("agent DB migrated: v{current} → v{SCHEMA_VERSION}"); - } - - Ok(()) -} - -fn get_schema_version(db: &Connection) -> Result { - let mut stmt = db - .prepare("SELECT value FROM _meta WHERE key = 'schema_version'") - .c(d!())?; - let mut rows = stmt.query([]).c(d!())?; - match rows.next().c(d!())? { - Some(row) => { - let val: String = row.get(0).c(d!())?; - val.parse::() - .map_err(|_| eg!("invalid schema_version: {val}")) - } - None => Ok(0), - } -} - -fn set_schema_version(db: &Connection, ver: u32) -> Result<()> { - db.execute( - "INSERT OR REPLACE INTO _meta (key, value) VALUES ('schema_version', ?1)", - rusqlite::params![ver.to_string()], - ) - .c(d!("set schema version"))?; - Ok(()) -} - -/// Load or generate a stable host_id persisted in the agent database. -/// -/// If `--host-id` is provided on the CLI, that value takes precedence and -/// is saved for future restarts. Otherwise we check the database; only -/// when neither is available do we generate a new random ID. -pub fn resolve_host_id(db_path: &str, cli_id: Option) -> Result { - let conn = Connection::open(db_path).c(d!("open DB for host_id"))?; - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS _meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - );", - ) - .c(d!("ensure meta table"))?; - - if let Some(id) = cli_id { - // CLI takes precedence — persist it - conn.execute( - "INSERT OR REPLACE INTO _meta (key, value) VALUES ('host_id', ?1)", - rusqlite::params![id], - ) - .c(d!("persist CLI host_id"))?; - return Ok(id); - } - - // Try to load from DB - let mut stmt = conn - .prepare("SELECT value FROM _meta WHERE key = 'host_id'") - .c(d!())?; - let mut rows = stmt.query([]).c(d!())?; - if let Some(row) = rows.next().c(d!())? { - let val: String = row.get(0).c(d!())?; - return Ok(val); - } - drop(rows); - drop(stmt); - - // Generate and persist a new ID - let id = uuid::Uuid::new_v4().to_string()[..8].to_string(); - conn.execute( - "INSERT OR REPLACE INTO _meta (key, value) VALUES ('host_id', ?1)", - rusqlite::params![id], - ) - .c(d!("persist generated host_id"))?; - Ok(id) -} - -fn save_vm(db: &Connection, vm: &Vm) -> Result<()> { - let data = serde_json::to_string(vm).c(d!("serialize VM"))?; - db.execute( - "INSERT OR REPLACE INTO vms (id, data) VALUES (?1, ?2)", - rusqlite::params![vm.id, data], - ) - .c(d!("save VM"))?; - Ok(()) -} - -fn load_vm(db: &Connection, id: &str) -> Result> { - let mut stmt = db - .prepare("SELECT data FROM vms WHERE id = ?1") - .c(d!("prepare load VM"))?; - let mut rows = stmt.query(rusqlite::params![id]).c(d!("query VM"))?; - match rows.next().c(d!("next row"))? { - Some(row) => { - let data: String = row.get(0).c(d!("get data"))?; - let vm: Vm = serde_json::from_str(&data).c(d!("deserialize VM"))?; - Ok(Some(vm)) - } - None => Ok(None), - } -} - -fn load_all_vms(db: &Connection) -> Result> { - let mut stmt = db - .prepare("SELECT data FROM vms") - .c(d!("prepare list VMs"))?; - let rows = stmt - .query_map([], |row| row.get::<_, String>(0)) - .c(d!("query all VMs"))?; - let mut vms = Vec::new(); - for row in rows { - let data = row.c(d!("read row"))?; - let vm: Vm = serde_json::from_str(&data).c(d!("deserialize VM"))?; - vms.push(vm); - } - Ok(vms) -} - -fn delete_vm(db: &Connection, id: &str) -> Result<()> { - db.execute("DELETE FROM vms WHERE id = ?1", rusqlite::params![id]) - .c(d!("delete VM"))?; - Ok(()) -} - -// ── Helpers ───────────────────────────────────────────────────────── - -fn now() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} - -fn detect_engines() -> Vec { - let mut engines = Vec::new(); - - #[cfg(target_os = "linux")] - { - if which("qemu-system-x86_64") { - engines.push(Engine::Qemu); - } - if which("firecracker") { - engines.push(Engine::Firecracker); - } - } - - #[cfg(target_os = "freebsd")] - { - if which("bhyve") { - engines.push(Engine::Bhyve); - } - if which("jail") { - engines.push(Engine::Jail); - } - } - - // Docker/Podman is available on all platforms - if which("docker") || which("podman") { - engines.push(Engine::Docker); - } - - engines -} - -fn which(cmd: &str) -> bool { - std::process::Command::new("which") - .arg(cmd) - .output() - .map(|o| o.status.success()) - .unwrap_or(false) -} - -fn ip_to_index(ip: &str) -> Option { - let parts: Vec<&str> = ip.split('.').collect(); - if parts.len() != 4 { - return None; - } - let hi: u32 = parts[2].parse().ok()?; - let lo: u32 = parts[3].parse().ok()?; - // Inverse of vm_ip: internal = hi*254 + (lo-1), index = internal - 1 - let internal = hi * 254 + lo.checked_sub(1)?; - internal.checked_sub(1) -} - -#[cfg(test)] -mod tests { - use super::*; - use ttcore::net; - - #[test] - fn ip_to_index_first() { - let ip = net::vm_ip(0); - assert_eq!(ip_to_index(&ip), Some(0)); - } - - #[test] - fn ip_to_index_sequential() { - for i in 0..100 { - let ip = net::vm_ip(i); - assert_eq!(ip_to_index(&ip), Some(i), "mismatch at index {i}: ip={ip}"); - } - } - - #[test] - fn ip_to_index_cross_octet() { - let ip = net::vm_ip(253); - let roundtrip = ip_to_index(&ip); - assert_eq!(roundtrip, Some(253), "ip={ip} roundtrip={roundtrip:?}"); - } - - #[test] - fn ip_to_index_invalid() { - assert_eq!(ip_to_index("not-an-ip"), None); - assert_eq!(ip_to_index("10.10.0"), None); - assert_eq!(ip_to_index("10.10.x.1"), None); - } - - // ── SQLite DB operations ──────────────────────────────────────── - - fn test_db() -> Connection { - let db = Connection::open(":memory:").unwrap(); - init_db(&db).unwrap(); - db - } - - fn make_vm(id: &str, state: VmState) -> Vm { - Vm { - id: id.into(), - env_id: "env1".into(), - host_id: "h1".into(), - image: "ubuntu".into(), - engine: Engine::Qemu, - cpu: 2, - mem: 1024, - disk: 40960, - ip: "10.10.0.2".into(), - port_map: BTreeMap::new(), - state, - created_at: 1000, - } - } - - #[test] - fn db_save_and_load_vm() { - let db = test_db(); - let vm = make_vm("vm1", VmState::Running); - save_vm(&db, &vm).unwrap(); - - let loaded = load_vm(&db, "vm1").unwrap().unwrap(); - assert_eq!(loaded.id, "vm1"); - assert_eq!(loaded.state, VmState::Running); - assert_eq!(loaded.cpu, 2); - } - - #[test] - fn db_load_nonexistent_vm() { - let db = test_db(); - let result = load_vm(&db, "nope").unwrap(); - assert!(result.is_none()); - } - - #[test] - fn db_load_all_vms() { - let db = test_db(); - save_vm(&db, &make_vm("a", VmState::Running)).unwrap(); - save_vm(&db, &make_vm("b", VmState::Stopped)).unwrap(); - let all = load_all_vms(&db).unwrap(); - assert_eq!(all.len(), 2); - } - - #[test] - fn db_delete_vm() { - let db = test_db(); - save_vm(&db, &make_vm("vm1", VmState::Running)).unwrap(); - delete_vm(&db, "vm1").unwrap(); - assert!(load_vm(&db, "vm1").unwrap().is_none()); - } - - #[test] - fn db_delete_nonexistent_vm() { - let db = test_db(); - // Should not error - delete_vm(&db, "nope").unwrap(); - } - - #[test] - fn db_upsert_vm() { - let db = test_db(); - let mut vm = make_vm("vm1", VmState::Creating); - save_vm(&db, &vm).unwrap(); - - vm.state = VmState::Running; - save_vm(&db, &vm).unwrap(); - - let loaded = load_vm(&db, "vm1").unwrap().unwrap(); - assert_eq!(loaded.state, VmState::Running); - // Only one row - assert_eq!(load_all_vms(&db).unwrap().len(), 1); - } - - #[test] - fn db_schema_version_persisted() { - let db = test_db(); - let ver = get_schema_version(&db).unwrap(); - assert_eq!(ver, SCHEMA_VERSION); - } - - // ── resolve_host_id ──────────────────────────────────────────── - - #[test] - fn resolve_host_id_generates_and_persists() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("test.db"); - let path_str = path.to_str().unwrap(); - - let id1 = resolve_host_id(path_str, None).unwrap(); - assert!(!id1.is_empty()); - assert!(id1.len() <= 8); - - // Second call returns the same persisted ID - let id2 = resolve_host_id(path_str, None).unwrap(); - assert_eq!(id1, id2); - } - - #[test] - fn resolve_host_id_cli_overrides() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("test.db"); - let path_str = path.to_str().unwrap(); - - let id1 = resolve_host_id(path_str, None).unwrap(); - let id2 = resolve_host_id(path_str, Some("custom-id".into())).unwrap(); - assert_eq!(id2, "custom-id"); - assert_ne!(id1, id2); - - // Persisted the CLI override - let id3 = resolve_host_id(path_str, None).unwrap(); - assert_eq!(id3, "custom-id"); - } -} diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml deleted file mode 100644 index f84a4c3..0000000 --- a/crates/cli/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "tt" -description = "TTstack CLI — manage your private cloud from the command line." -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true - -[[bin]] -name = "tt" -path = "src/main.rs" - -[dependencies] -ttcore = { path = "../core" } -serde = { workspace = true } -serde_json = { workspace = true } -ruc = { workspace = true } -tokio = { workspace = true } -reqwest = { workspace = true } -clap = { workspace = true } -toml = { workspace = true } -uuid = { workspace = true } diff --git a/crates/cli/src/client.rs b/crates/cli/src/client.rs deleted file mode 100644 index 0223881..0000000 --- a/crates/cli/src/client.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! HTTP client for communicating with the tt-ctl controller. - -use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; -use ruc::*; -use serde::Serialize; -use serde::de::DeserializeOwned; -use ttcore::api::ApiResp; - -/// Controller API client. -pub struct Client { - base_url: String, - http: reqwest::Client, -} - -impl Client { - pub fn new(addr: &str, api_key: Option<&str>) -> Self { - let base_url = if addr.starts_with("http") { - addr.to_string() - } else { - format!("http://{addr}") - }; - - let mut headers = HeaderMap::new(); - if let Some(key) = api_key - && let Ok(val) = HeaderValue::from_str(&format!("Bearer {key}")) - { - headers.insert(AUTHORIZATION, val); - } - - Self { - base_url, - http: reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(60)) - .default_headers(headers) - .build() - .unwrap(), - } - } - - /// GET request, returning deserialized data. - pub async fn get(&self, path: &str) -> Result { - let url = format!("{}{path}", self.base_url); - let resp = self.http.get(&url).send().await.c(d!("request failed"))?; - let status = resp.status(); - let body: ApiResp = resp.json().await.c(d!("invalid response"))?; - - if body.ok { - body.data.ok_or_else(|| eg!("empty response")) - } else { - Err(eg!(body.error.unwrap_or_else(|| format!("HTTP {status}")))) - } - } - - /// POST request with JSON body, returning deserialized data. - pub async fn post(&self, path: &str, body: &B) -> Result { - let url = format!("{}{path}", self.base_url); - let resp = self - .http - .post(&url) - .json(body) - .send() - .await - .c(d!("request failed"))?; - let status = resp.status(); - let body: ApiResp = resp.json().await.c(d!("invalid response"))?; - - if body.ok { - body.data.ok_or_else(|| eg!("empty response")) - } else { - Err(eg!(body.error.unwrap_or_else(|| format!("HTTP {status}")))) - } - } - - /// POST request with no request body, no response body. - pub async fn post_action(&self, path: &str) -> Result<()> { - let url = format!("{}{path}", self.base_url); - let resp = self.http.post(&url).send().await.c(d!("request failed"))?; - let status = resp.status(); - let body: ApiResp<()> = resp.json().await.c(d!("invalid response"))?; - - if body.ok { - Ok(()) - } else { - Err(eg!(body.error.unwrap_or_else(|| format!("HTTP {status}")))) - } - } - - /// DELETE request. - pub async fn delete(&self, path: &str) -> Result<()> { - let url = format!("{}{path}", self.base_url); - let resp = self - .http - .delete(&url) - .send() - .await - .c(d!("request failed"))?; - let status = resp.status(); - let body: ApiResp<()> = resp.json().await.c(d!("invalid response"))?; - - if body.ok { - Ok(()) - } else { - Err(eg!(body.error.unwrap_or_else(|| format!("HTTP {status}")))) - } - } -} - -// ── Configuration File ────────────────────────────────────────────── - -const CONFIG_FILE: &str = ".ttconfig"; - -/// CLI configuration: controller address and optional API key. -pub struct CliConfig { - pub addr: String, - pub api_key: Option, -} - -/// Read the CLI config from ~/.ttconfig. -/// -/// File format (one value per line): -/// ```text -/// -/// # optional second line -/// ``` -pub fn load_config() -> Option { - let path = dirs_path(); - let content = std::fs::read_to_string(&path).ok()?; - let mut lines = content.lines(); - let addr = lines.next()?.trim().to_string(); - if addr.is_empty() { - return None; - } - let api_key = lines - .next() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()); - Some(CliConfig { addr, api_key }) -} - -/// Save the controller address and optional API key to ~/.ttconfig. -pub fn save_config(addr: &str, api_key: Option<&str>) -> Result<()> { - let path = dirs_path(); - let content = match api_key { - Some(key) => format!("{addr}\n{key}\n"), - None => format!("{addr}\n"), - }; - std::fs::write(&path, &content).c(d!("save config"))?; - - // Restrict file permissions to owner-only (0600) - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o600); - let _ = std::fs::set_permissions(&path, perms); - } - - Ok(()) -} - -fn dirs_path() -> String { - let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); - format!("{home}/{CONFIG_FILE}") -} diff --git a/crates/cli/src/deploy.rs b/crates/cli/src/deploy.rs deleted file mode 100644 index b342a9c..0000000 --- a/crates/cli/src/deploy.rs +++ /dev/null @@ -1,823 +0,0 @@ -//! Distributed deployment — Rust implementation. -//! -//! Replaces `tools/deploy.sh` with a reliable, idempotent deploy -//! embedded directly in the `tt` CLI binary. -//! -//! Supports both systemd (Debian/Ubuntu/Rocky) and OpenRC (Alpine) -//! init systems, and handles non-root SSH users via sudo. - -use ruc::*; -use serde::Deserialize; -use std::path::{Path, PathBuf}; -use tokio::process::Command; - -// ── Deploy config (deploy.toml) ───────────────────────────────────── - -#[derive(Debug, Deserialize)] -pub struct DeployConfig { - #[serde(default)] - pub general: GeneralConfig, - #[serde(default)] - pub controller: Option, - #[serde(default)] - pub agents: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct GeneralConfig { - #[serde(default = "default_prefix")] - pub prefix: String, - #[serde(default = "default_user")] - pub user: String, - #[serde(default = "default_release_dir")] - pub release_dir: String, - /// API key for controller authentication. If not set, a random key is generated. - pub api_key: Option, -} - -impl Default for GeneralConfig { - fn default() -> Self { - Self { - prefix: default_prefix(), - user: default_user(), - release_dir: default_release_dir(), - api_key: None, - } - } -} - -fn generate_api_key() -> String { - format!( - "tt-{}{}", - uuid::Uuid::new_v4().simple(), - uuid::Uuid::new_v4().simple() - ) -} - -#[derive(Debug, Deserialize)] -pub struct ControllerConfig { - pub host: String, - #[serde(default = "default_ssh_user")] - pub ssh_user: String, - #[serde(default = "default_ssh_port")] - pub ssh_port: u16, - #[serde(default = "default_ctl_listen")] - pub listen: String, - pub data_dir: Option, -} - -#[derive(Debug, Deserialize)] -pub struct AgentConfig { - pub host: String, - #[serde(default = "default_ssh_user")] - pub ssh_user: String, - #[serde(default = "default_ssh_port")] - pub ssh_port: u16, - #[serde(default = "default_agent_listen")] - pub listen: String, - #[serde(default = "default_storage")] - pub storage: String, - pub image_dir: Option, - pub runtime_dir: Option, - #[serde(default)] - pub cpu_total: u32, - #[serde(default)] - pub mem_total: u32, - #[serde(default = "default_disk_total")] - pub disk_total: String, - pub host_id: Option, - /// Override release_dir for this agent (e.g. for musl/cross-compiled binaries). - pub release_dir: Option, -} - -fn default_prefix() -> String { - "/opt/ttstack".into() -} -fn default_user() -> String { - "ttstack".into() -} -fn default_release_dir() -> String { - "./target/release".into() -} -fn default_ssh_user() -> String { - "root".into() -} -fn default_ssh_port() -> u16 { - 22 -} -fn default_ctl_listen() -> String { - "0.0.0.0:9200".into() -} -fn default_agent_listen() -> String { - "0.0.0.0:9100".into() -} -fn default_storage() -> String { - "file".into() -} -fn default_disk_total() -> String { - "200G".into() -} - -/// Parse disk_total: "200G" → 204800 (MiB), plain number passes through. -fn parse_disk(val: &str) -> u32 { - let val = val.trim().trim_matches('"'); - if let Some(num) = val.strip_suffix(['g', 'G']) { - num.parse::().unwrap_or(200) * 1024 - } else { - val.parse().unwrap_or(204800) - } -} - -// ── SSH helpers ───────────────────────────────────────────────────── - -struct SshTarget { - user: String, - host: String, - port: u16, -} - -impl SshTarget { - async fn exec(&self, cmd: &str) -> Result { - let output = Command::new("ssh") - .args([ - "-o", - "StrictHostKeyChecking=no", - "-o", - "ConnectTimeout=10", - "-p", - &self.port.to_string(), - ]) - .arg(format!("{}@{}", self.user, self.host)) - .arg(cmd) - .output() - .await - .c(d!("ssh exec failed"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("ssh command failed on {}: {}", self.host, stderr)); - } - Ok(String::from_utf8_lossy(&output.stdout).to_string()) - } - - async fn copy(&self, local: &Path, remote: &str) -> Result<()> { - let output = Command::new("scp") - .args([ - "-o", - "StrictHostKeyChecking=no", - "-o", - "ConnectTimeout=10", - "-P", - &self.port.to_string(), - ]) - .arg(local) - .arg(format!("{}@{}:{}", self.user, self.host, remote)) - .output() - .await - .c(d!("scp failed"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("scp to {} failed: {}", self.host, stderr)); - } - Ok(()) - } -} - -// ── Service templates ─────────────────────────────────────────────── - -fn systemd_unit(name: &str, exec_start: &str, run_as_root: bool, env_file: Option<&str>) -> String { - let user_lines = if run_as_root { - "# Runs as root (needs NET_ADMIN for bridge/TAP/nftables)".to_string() - } else { - "User=ttstack\nGroup=ttstack".to_string() - }; - - let env_file_line = match env_file { - Some(path) => format!("EnvironmentFile={path}\n"), - None => String::new(), - }; - - format!( - r#"[Unit] -Description=TTstack {name} -After=network.target - -[Service] -Type=simple -{user_lines} -{env_file_line}ExecStart={exec_start} -Restart=on-failure -RestartSec=5 -LimitNOFILE=65536 - -[Install] -WantedBy=multi-user.target -"# - ) -} - -fn openrc_initd(name: &str, exec_start: &str) -> String { - // Split exec_start into command and args for OpenRC - let mut parts = exec_start.splitn(2, ' '); - let cmd = parts.next().unwrap_or(exec_start); - let args = parts.next().unwrap_or(""); - - format!( - r#"#!/sbin/openrc-run - -name="{name}" -description="TTstack {name}" -command="{cmd}" -command_args="{args}" -command_background=true -pidfile="/run/${{name}}.pid" -output_log="/var/log/${{name}}.log" -error_log="/var/log/${{name}}.log" - -depend() {{ - need net - after firewall -}} -"# - ) -} - -/// Generate the platform-aware remote setup script. -/// -/// Uses `sudo` for all privileged operations, detects init system, -/// and handles both glibc (useradd) and busybox (adduser) user creation. -#[allow(clippy::too_many_arguments)] -fn remote_setup_script( - service_name: &str, - exec_cmd: &str, - user: &str, - home: &str, - prefix: &str, - tmp: &str, - binaries: &[&str], - extra_dirs: &[&str], - host_label: &str, - env_file: Option<(&str, &str)>, -) -> String { - let bin_copies: String = binaries - .iter() - .map(|b| format!("sudo cp {tmp}/{b} {prefix}/bin/{b}\nsudo chmod 755 {prefix}/bin/{b}",)) - .collect::>() - .join("\n"); - - let dir_list: String = extra_dirs - .iter() - .map(|d| d.to_string()) - .chain([format!("{home}/data"), format!("{home}/run")]) - .collect::>() - .join(" "); - - let env_file_path = env_file.map(|(p, _)| p); - let systemd_unit_content = systemd_unit(service_name, exec_cmd, true, env_file_path); - let openrc_initd_content = openrc_initd(service_name, exec_cmd); - - // Shell snippet that writes the env file with 0600 permissions - let env_file_setup = match env_file { - Some((path, content)) => format!( - r#" -# Write environment file (keeps secrets out of process args) -sudo mkdir -p $(dirname {path}) -printf '%s' '{content}' | sudo tee {path} > /dev/null -sudo chmod 600 {path} -"#, - path = path, - content = content, - ), - None => String::new(), - }; - - // For OpenRC / manual fallback, source env file before exec - let env_source = match env_file { - Some((path, _)) => format!(". {path} && export TT_API_KEY && "), - None => String::new(), - }; - - format!( - r#"#!/bin/sh -set -e - -# Create service user (portable: works on glibc and busybox) -if id {user} >/dev/null 2>&1; then - echo "[deploy] user '{user}' exists" -else - echo "[deploy] creating user '{user}'" - if command -v useradd >/dev/null 2>&1; then - sudo useradd -r -m -d {home} -s /bin/sh {user} - elif command -v adduser >/dev/null 2>&1; then - sudo adduser -D -h {home} -s /bin/sh {user} 2>/dev/null || true - fi -fi - -# Create directories -sudo mkdir -p {prefix}/bin {prefix}/etc {dir_list} - -# Install binaries -{bin_copies} - -# Set ownership -sudo chown -R {user}:{user} {home} 2>/dev/null || true -{env_file_setup} -# Detect init system and install service -if command -v systemctl >/dev/null 2>&1 && [ -d /etc/systemd/system ]; then - echo "[deploy] using systemd" - sudo tee /etc/systemd/system/{service_name}.service > /dev/null <<'UNIT' -{systemd_unit} -UNIT - sudo systemctl daemon-reload - sudo systemctl enable {service_name} - sudo systemctl restart {service_name} - sleep 1 - sudo systemctl is-active {service_name} && echo "[deploy] {service_name} is active" -elif command -v rc-service >/dev/null 2>&1; then - echo "[deploy] using OpenRC" - sudo tee /etc/init.d/{service_name} > /dev/null <<'INITD' -{openrc_initd} -INITD - sudo chmod 755 /etc/init.d/{service_name} - sudo rc-update add {service_name} default 2>/dev/null || true - sudo rc-service {service_name} restart 2>/dev/null || sudo rc-service {service_name} start - sleep 1 - sudo rc-service {service_name} status && echo "[deploy] {service_name} is running" -else - echo "[deploy] WARNING: no known init system; starting {service_name} manually" - sudo pkill -f '{prefix}/bin/{service_name}' 2>/dev/null || true - sleep 1 - sudo nohup sh -c '{env_source}{exec_cmd}' > /var/log/{service_name}.log 2>&1 & - sleep 2 - pgrep -f '{prefix}/bin/{service_name}' && echo "[deploy] {service_name} started (manual)" -fi - -# Cleanup -rm -rf {tmp} -echo "[deploy] {service_name} deployed on {host_label}" -"#, - user = user, - home = home, - prefix = prefix, - tmp = tmp, - dir_list = dir_list, - bin_copies = bin_copies, - service_name = service_name, - exec_cmd = exec_cmd, - systemd_unit = systemd_unit_content, - openrc_initd = openrc_initd_content, - host_label = host_label, - env_file_setup = env_file_setup, - env_source = env_source, - ) -} - -// ── Local deploy ──────────────────────────────────────────────────── - -async fn local_ensure_user(user: &str) -> Result<()> { - let check = Command::new("id") - .arg(user) - .output() - .await - .c(d!("check user"))?; - - if check.status.success() { - println!("[deploy] user '{user}' exists"); - } else { - println!("[deploy] creating user '{user}'"); - let out = Command::new("useradd") - .args([ - "-r", - "-m", - "-d", - &format!("/home/{user}"), - "-s", - "/bin/sh", - user, - ]) - .output() - .await - .c(d!("create user"))?; - if !out.status.success() { - let stderr = String::from_utf8_lossy(&out.stderr); - return Err(eg!("useradd failed: {}", stderr)); - } - } - Ok(()) -} - -async fn local_ensure_dirs(home: &str, user: &str) -> Result<()> { - for dir in ["images", "runtime", "data", "ctl", "run"] { - let path = format!("{home}/{dir}"); - tokio::fs::create_dir_all(&path).await.c(d!("mkdir"))?; - } - Command::new("chown") - .args(["-R", &format!("{user}:{user}"), home]) - .output() - .await - .c(d!("chown"))?; - Ok(()) -} - -async fn local_install_bin(src: &Path, prefix: &str) -> Result<()> { - let bin_dir = format!("{prefix}/bin"); - tokio::fs::create_dir_all(&bin_dir) - .await - .c(d!("mkdir bin"))?; - - let name = src.file_name().unwrap().to_str().unwrap(); - let dst = format!("{bin_dir}/{name}"); - tokio::fs::copy(src, &dst).await.c(d!("copy binary"))?; - - Command::new("chmod") - .args(["755", &dst]) - .output() - .await - .c(d!("chmod"))?; - - println!("[deploy] installed {dst}"); - Ok(()) -} - -async fn local_install_systemd( - name: &str, - exec_start: &str, - run_as_root: bool, - env_file: Option<(&str, &str)>, -) -> Result<()> { - // Write optional environment file with restricted permissions - if let Some((path, content)) = env_file { - if let Some(parent) = std::path::Path::new(path).parent() { - tokio::fs::create_dir_all(parent) - .await - .c(d!("create env dir"))?; - } - tokio::fs::write(path, content) - .await - .c(d!("write env file"))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o600); - let _ = std::fs::set_permissions(path, perms); - } - } - - let env_path = env_file.map(|(p, _)| p); - let unit = systemd_unit(name, exec_start, run_as_root, env_path); - let path = format!("/etc/systemd/system/{name}.service"); - tokio::fs::write(&path, &unit).await.c(d!("write unit"))?; - - Command::new("systemctl") - .arg("daemon-reload") - .output() - .await - .c(d!("daemon-reload"))?; - Command::new("systemctl") - .args(["enable", name]) - .output() - .await - .c(d!("enable service"))?; - - println!("[deploy] unit {name} installed"); - Ok(()) -} - -async fn local_restart_service(name: &str) -> Result<()> { - let active = Command::new("systemctl") - .args(["is-active", "--quiet", name]) - .output() - .await; - - let action = if active.map(|o| o.status.success()).unwrap_or(false) { - "restart" - } else { - "start" - }; - - Command::new("systemctl") - .args([action, name]) - .output() - .await - .c(d!("restart service"))?; - - let status = Command::new("systemctl") - .args(["is-active", name]) - .output() - .await - .c(d!("check status"))?; - let state = String::from_utf8_lossy(&status.stdout); - println!("[deploy] {name} is {}", state.trim()); - Ok(()) -} - -// ── Deploy entry points ───────────────────────────────────────────── - -/// Deploy locally on this host (requires root). -pub async fn deploy_local(role: &str, release_dir: &str) -> Result<()> { - let uid = std::fs::read_to_string("/proc/self/status") - .ok() - .and_then(|s| { - s.lines() - .find(|l| l.starts_with("Uid:")) - .and_then(|l| l.split_whitespace().nth(1)) - .and_then(|v| v.parse::().ok()) - }) - .unwrap_or(1000); - if uid != 0 { - return Err(eg!("local deploy requires root (run with sudo)")); - } - - let prefix = "/opt/ttstack"; - let user = "ttstack"; - let home = "/home/ttstack"; - - local_ensure_user(user).await?; - local_ensure_dirs(home, user).await?; - - let release = PathBuf::from(release_dir); - let api_key = generate_api_key(); - let env_content = format!("TT_API_KEY={api_key}\n"); - - match role { - "agent" | "all" => { - let bin = release.join("tt-agent"); - if !bin.exists() { - return Err(eg!( - "{} not found (run 'cargo build --release' first)", - bin.display() - )); - } - local_install_bin(&bin, prefix).await?; - - let cmd = format!( - "{prefix}/bin/tt-agent --listen 0.0.0.0:9100 \ - --image-dir {home}/images --runtime-dir {home}/runtime \ - --data-dir {home}/data --storage file" - ); - let env_path = format!("{prefix}/etc/tt-agent.env"); - local_install_systemd("tt-agent", &cmd, true, Some((&env_path, &env_content))).await?; - local_restart_service("tt-agent").await?; - } - _ => {} - } - - match role { - "ctl" | "all" => { - for name in ["tt-ctl", "tt"] { - let bin = release.join(name); - if !bin.exists() { - return Err(eg!("{} not found", bin.display())); - } - local_install_bin(&bin, prefix).await?; - } - - let cmd = format!("{prefix}/bin/tt-ctl --listen 0.0.0.0:9200 --data-dir {home}/ctl"); - let env_path = format!("{prefix}/etc/tt-ctl.env"); - local_install_systemd("tt-ctl", &cmd, false, Some((&env_path, &env_content))).await?; - local_restart_service("tt-ctl").await?; - - println!("[deploy] API key: {api_key}"); - println!("[deploy] Run: tt config 127.0.0.1:9200 --api-key {api_key}"); - } - _ => {} - } - - println!("[deploy] local deploy complete"); - Ok(()) -} - -/// Distributed deploy from a config file. -pub async fn deploy_distributed(config_path: &str) -> Result<()> { - let content = std::fs::read_to_string(config_path).c(d!("read config"))?; - let cfg: DeployConfig = - toml::from_str(&content).map_err(|e| eg!(format!("parse deploy.toml: {e}")))?; - - let release_dir = PathBuf::from(&cfg.general.release_dir); - let prefix = &cfg.general.prefix; - let user = &cfg.general.user; - let home = format!("/home/{user}"); - - // Verify local binaries exist - for bin in ["tt", "tt-ctl", "tt-agent"] { - let p = release_dir.join(bin); - if !p.exists() { - return Err(eg!( - "{} not found (run 'cargo build --release')", - p.display() - )); - } - } - - let api_key = cfg.general.api_key.clone().unwrap_or_else(generate_api_key); - let env_content = format!("TT_API_KEY={api_key}\n"); - - // Deploy controller - if let Some(ctl) = &cfg.controller { - println!("\n[deploy] === {} (controller) ===", ctl.host); - let target = SshTarget { - user: ctl.ssh_user.clone(), - host: ctl.host.clone(), - port: ctl.ssh_port, - }; - - let tmp = format!("/tmp/ttstack-deploy-{}", std::process::id()); - target.exec(&format!("mkdir -p {tmp}")).await?; - - for bin in ["tt-ctl", "tt"] { - target - .copy(&release_dir.join(bin), &format!("{tmp}/{bin}")) - .await?; - println!("[deploy] uploaded {bin}"); - } - - let data_dir = ctl - .data_dir - .clone() - .unwrap_or_else(|| format!("{home}/ctl")); - let exec_cmd = format!( - "{prefix}/bin/tt-ctl --listen {listen} --data-dir {data_dir}", - listen = ctl.listen, - ); - let env_path = format!("{prefix}/etc/tt-ctl.env"); - - let script = remote_setup_script( - "tt-ctl", - &exec_cmd, - user, - &home, - prefix, - &tmp, - &["tt-ctl", "tt"], - &[format!("{home}/ctl").as_str()], - &ctl.host, - Some((&env_path, &env_content)), - ); - let out = target.exec(&script).await?; - print!("{out}"); - } - - // Deploy agents - for (i, agent) in cfg.agents.iter().enumerate() { - println!("\n[deploy] === {} (agent {}) ===", agent.host, i); - let target = SshTarget { - user: agent.ssh_user.clone(), - host: agent.host.clone(), - port: agent.ssh_port, - }; - - let tmp = format!("/tmp/ttstack-deploy-{}", std::process::id()); - target.exec(&format!("mkdir -p {tmp}")).await?; - - let agent_release = agent - .release_dir - .as_ref() - .map(PathBuf::from) - .unwrap_or_else(|| release_dir.clone()); - target - .copy(&agent_release.join("tt-agent"), &format!("{tmp}/tt-agent")) - .await?; - println!("[deploy] uploaded tt-agent"); - - let image_dir = agent - .image_dir - .clone() - .unwrap_or_else(|| format!("{home}/images")); - let runtime_dir = agent - .runtime_dir - .clone() - .unwrap_or_else(|| format!("{home}/runtime")); - let disk = parse_disk(&agent.disk_total); - - let mut exec_cmd = format!( - "{prefix}/bin/tt-agent --listen {listen} \ - --image-dir {image_dir} --runtime-dir {runtime_dir} \ - --data-dir {home}/data --storage {storage} \ - --cpu-total {cpu} --mem-total {mem} --disk-total {disk}", - listen = agent.listen, - storage = agent.storage, - cpu = agent.cpu_total, - mem = agent.mem_total, - ); - if let Some(hid) = &agent.host_id { - exec_cmd.push_str(&format!(" --host-id {hid}")); - } - let env_path = format!("{prefix}/etc/tt-agent.env"); - - let script = remote_setup_script( - "tt-agent", - &exec_cmd, - user, - &home, - prefix, - &tmp, - &["tt-agent"], - &[image_dir.as_str(), runtime_dir.as_str()], - &agent.host, - Some((&env_path, &env_content)), - ); - let out = target.exec(&script).await?; - print!("{out}"); - } - - println!("\n[deploy] distributed deployment complete"); - println!("[deploy] API key: {api_key}"); - if let Some(ctl) = &cfg.controller { - println!( - "[deploy] Run: tt config {}:{} --api-key {api_key}", - ctl.host, - ctl.listen.rsplit(':').next().unwrap_or("9200") - ); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_disk_gib() { - assert_eq!(parse_disk("200G"), 204800); - assert_eq!(parse_disk("100g"), 102400); - assert_eq!(parse_disk("\"500G\""), 512000); - } - - #[test] - fn parse_disk_mib() { - assert_eq!(parse_disk("204800"), 204800); - assert_eq!(parse_disk("1024"), 1024); - } - - #[test] - fn parse_config_minimal() { - let toml_str = r#" -[[agents]] -host = "10.0.0.2" -"#; - let cfg: DeployConfig = toml::from_str(toml_str).unwrap(); - assert!(cfg.controller.is_none()); - assert_eq!(cfg.agents.len(), 1); - assert_eq!(cfg.agents[0].host, "10.0.0.2"); - assert_eq!(cfg.agents[0].ssh_port, 22); - assert_eq!(cfg.agents[0].storage, "file"); - } - - #[test] - fn parse_config_full() { - let toml_str = r#" -[general] -prefix = "/opt/tt" -user = "myuser" -api_key = "my-secret-key" - -[controller] -host = "10.0.0.1" -listen = "0.0.0.0:9200" - -[[agents]] -host = "10.0.0.2" -storage = "zvol" -host_id = "node-a" -cpu_total = 32 -mem_total = 65536 -disk_total = "1000G" -image_dir = "tank/images" -runtime_dir = "tank/runtime" - -[[agents]] -host = "10.0.0.3" -"#; - let cfg: DeployConfig = toml::from_str(toml_str).unwrap(); - assert_eq!(cfg.general.prefix, "/opt/tt"); - assert_eq!(cfg.general.api_key.as_deref(), Some("my-secret-key")); - assert!(cfg.controller.is_some()); - assert_eq!(cfg.agents.len(), 2); - assert_eq!(cfg.agents[0].storage, "zvol"); - assert_eq!(cfg.agents[0].host_id.as_deref(), Some("node-a")); - assert_eq!(parse_disk(&cfg.agents[0].disk_total), 1024000); - } - - #[test] - fn remote_script_contains_sudo() { - let script = remote_setup_script( - "tt-agent", - "/opt/tt/bin/tt-agent --listen 0.0.0.0:9100", - "ttstack", - "/home/ttstack", - "/opt/tt", - "/tmp/deploy-1", - &["tt-agent"], - &["/home/ttstack/images"], - "testhost", - Some(("/opt/tt/etc/tt-agent.env", "TT_API_KEY=test-key\n")), - ); - assert!(script.contains("sudo")); - assert!(script.contains("rc-service") || script.contains("systemctl")); - assert!(script.contains("adduser") || script.contains("useradd")); - assert!(script.contains("EnvironmentFile=/opt/tt/etc/tt-agent.env")); - assert!(script.contains("TT_API_KEY=test-key")); - assert!(!script.contains("--api-key")); - } -} diff --git a/crates/cli/src/image_builder.rs b/crates/cli/src/image_builder.rs deleted file mode 100644 index defb70c..0000000 --- a/crates/cli/src/image_builder.rs +++ /dev/null @@ -1,524 +0,0 @@ -//! Automatic guest image creation for all supported engines. -//! -//! Generates ready-to-use images so users can start creating VMs -//! immediately after deploying TTstack. - -use ruc::*; -use std::path::{Path, PathBuf}; -use tokio::process::Command; - -// ── Image catalog ─────────────────────────────────────────────────── - -/// A built-in image recipe that can be auto-generated. -pub struct ImageRecipe { - pub name: &'static str, - pub engine: &'static str, - pub description: &'static str, -} - -/// List of all auto-generatable images. -pub const RECIPES: &[ImageRecipe] = &[ - // Docker / Podman — lightweight containers - ImageRecipe { - name: "alpine", - engine: "docker", - description: "Alpine Linux 3.21 (minimal, ~8MB)", - }, - ImageRecipe { - name: "debian", - engine: "docker", - description: "Debian 13 Trixie (slim, ~75MB)", - }, - ImageRecipe { - name: "ubuntu", - engine: "docker", - description: "Ubuntu 24.04 LTS (minimal, ~30MB)", - }, - ImageRecipe { - name: "rockylinux", - engine: "docker", - description: "Rocky Linux 9 (minimal, ~70MB)", - }, - ImageRecipe { - name: "nginx", - engine: "docker", - description: "Nginx web server (Alpine-based, ~45MB)", - }, - ImageRecipe { - name: "redis", - engine: "docker", - description: "Redis 7 (Alpine-based, ~35MB)", - }, - ImageRecipe { - name: "postgres", - engine: "docker", - description: "PostgreSQL 17 (Alpine-based, ~85MB)", - }, - // Firecracker — microVMs - ImageRecipe { - name: "fc-alpine", - engine: "firecracker", - description: "Alpine Linux microVM (kernel + rootfs, ~50MB)", - }, - // QEMU/KVM — full VMs (cloud images) - ImageRecipe { - name: "alpine-cloud", - engine: "qemu", - description: "Alpine Linux 3.21 cloud image (qcow2, ~150MB)", - }, - ImageRecipe { - name: "debian-cloud", - engine: "qemu", - description: "Debian 13 generic cloud image (qcow2, ~350MB)", - }, - ImageRecipe { - name: "ubuntu-cloud", - engine: "qemu", - description: "Ubuntu 24.04 cloud image (qcow2, ~600MB)", - }, - // Jail — FreeBSD containers - ImageRecipe { - name: "freebsd-base", - engine: "jail", - description: "FreeBSD 14.3 base (fetched from releases, ~180MB)", - }, -]; - -/// Print available image recipes. -pub fn list_recipes() { - println!("{:<16} {:<14} DESCRIPTION", "NAME", "ENGINE"); - for r in RECIPES { - println!("{:<16} {:<14} {}", r.name, r.engine, r.description); - } -} - -// ── Docker / Podman images ────────────────────────────────────────── - -/// Map recipe name to Docker image tag. -fn docker_tag(name: &str) -> &str { - match name { - "alpine" => "alpine:3.21", - "debian" => "debian:trixie-slim", - "ubuntu" => "ubuntu:24.04", - "rockylinux" => "rockylinux:9-minimal", - "nginx" => "nginx:alpine", - "redis" => "redis:7-alpine", - "postgres" => "postgres:17-alpine", - _ => name, - } -} - -async fn detect_runtime() -> Result<&'static str> { - if Command::new("docker") - .arg("version") - .output() - .await - .map(|o| o.status.success()) - .unwrap_or(false) - { - return Ok("docker"); - } - if Command::new("podman") - .arg("version") - .output() - .await - .map(|o| o.status.success()) - .unwrap_or(false) - { - return Ok("podman"); - } - Err(eg!("neither docker nor podman found")) -} - -async fn create_docker(name: &str) -> Result<()> { - let rt = detect_runtime().await?; - let tag = docker_tag(name); - - println!("[image] pulling {tag} via {rt}..."); - let output = Command::new(rt) - .args(["pull", tag]) - .output() - .await - .c(d!("pull failed"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("{rt} pull {tag} failed: {}", stderr)); - } - - // Tag as the short name so `tt env create --image alpine` works - if tag != name { - let _ = Command::new(rt).args(["tag", tag, name]).output().await; - } - - println!("[image] {name} ready ({rt})"); - Ok(()) -} - -// ── Firecracker images ────────────────────────────────────────────── - -const FC_KERNEL_URL: &str = - "https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin"; - -/// Alpine rootfs mirror. -const ALPINE_MINIROOTFS_URL: &str = "https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/x86_64/alpine-minirootfs-3.21.3-x86_64.tar.gz"; - -async fn create_firecracker(name: &str, image_dir: &Path) -> Result<()> { - let target = image_dir.join(name); - tokio::fs::create_dir_all(&target).await.c(d!("mkdir"))?; - - let kernel = target.join("vmlinux"); - let rootfs = target.join("rootfs.ext4"); - - // Download kernel - if !kernel.exists() { - println!("[image] downloading Firecracker kernel..."); - download_file(FC_KERNEL_URL, &kernel).await?; - println!("[image] kernel: {}", human_size(&kernel).await); - } else { - println!("[image] kernel already exists"); - } - - // Create rootfs with Alpine userspace - if rootfs.exists() { - println!("[image] rootfs already exists"); - return Ok(()); - } - - let rootfs_mb: u32 = 128; - println!("[image] creating {rootfs_mb}MB rootfs with Alpine userspace..."); - - // Create empty ext4 image - run_cmd( - "dd", - &[ - "if=/dev/zero", - &format!("of={}", rootfs.display()), - "bs=1M", - &format!("count={rootfs_mb}"), - ], - ) - .await?; - run_cmd("mkfs.ext4", &["-q", &rootfs.display().to_string()]).await?; - - // Mount and populate - let mnt = tempdir()?; - run_cmd( - "mount", - &[ - "-o", - "loop", - &rootfs.display().to_string(), - &mnt.display().to_string(), - ], - ) - .await?; - - // Download and extract Alpine minirootfs - let tarball = format!("{}/alpine.tar.gz", mnt.display()); - download_file(ALPINE_MINIROOTFS_URL, Path::new(&tarball)).await?; - run_cmd("tar", &["xzf", &tarball, "-C", &mnt.display().to_string()]).await?; - tokio::fs::remove_file(&tarball).await.ok(); - - // Create init wrapper that mounts essential filesystems - let init_script = format!( - r#"#!/bin/sh -mount -t proc proc /proc -mount -t sysfs sysfs /sys -mount -t devtmpfs devtmpfs /dev 2>/dev/null -mkdir -p /dev/pts -mount -t devpts devpts /dev/pts -hostname ttstack -echo "TTstack Firecracker guest [{name}] booted OK" - -# Set up networking if virtio-net is available -ip link set eth0 up 2>/dev/null -ip addr add 10.10.0.2/16 dev eth0 2>/dev/null -ip route add default via 10.10.0.1 2>/dev/null - -# Start shell or sleep forever -if [ -x /bin/sh ]; then - /bin/sh -else - while true; do sleep 3600; done -fi -"# - ); - let init_path = format!("{}/init", mnt.display()); - tokio::fs::write(&init_path, init_script) - .await - .c(d!("write init"))?; - run_cmd("chmod", &["755", &init_path]).await?; - - // Ensure /sbin/init symlink - let sbin = format!("{}/sbin", mnt.display()); - tokio::fs::create_dir_all(&sbin).await.ok(); - let sbin_init = format!("{sbin}/init"); - if !Path::new(&sbin_init).exists() { - tokio::fs::symlink("/init", &sbin_init).await.ok(); - } - - // Set up DNS - let etc = format!("{}/etc", mnt.display()); - tokio::fs::create_dir_all(&etc).await.ok(); - tokio::fs::write(format!("{etc}/resolv.conf"), "nameserver 8.8.8.8\n") - .await - .ok(); - - run_cmd("umount", &[&mnt.display().to_string()]).await?; - tokio::fs::remove_dir(&mnt).await.ok(); - - println!( - "[image] {name} ready: kernel={}, rootfs={}", - human_size(&kernel).await, - human_size(&rootfs).await - ); - Ok(()) -} - -// ── QEMU cloud images ────────────────────────────────────────────── - -fn qemu_cloud_url(name: &str) -> Option<&'static str> { - match name { - "alpine-cloud" => Some( - "https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/cloud/nocloud_alpine-3.21.3-x86_64-bios-cloudinit-r0.qcow2", - ), - "debian-cloud" => Some( - "https://cloud.debian.org/images/cloud/trixie/daily/latest/debian-13-generic-amd64-daily.qcow2", - ), - "ubuntu-cloud" => { - Some("https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img") - } - _ => None, - } -} - -async fn create_qemu(name: &str, image_dir: &Path) -> Result<()> { - let target = image_dir.join(name); - - if target.exists() { - println!("[image] {name} already exists"); - return Ok(()); - } - - let url = qemu_cloud_url(name).ok_or_else(|| eg!("unknown QEMU image: {name}"))?; - - println!("[image] downloading {name} cloud image..."); - download_file(url, &target).await?; - - // Ensure it's qcow2 format - let ext = url.rsplit('.').next().unwrap_or(""); - if ext == "img" { - // Convert raw to qcow2 - let tmp = target.with_extension("raw"); - tokio::fs::rename(&target, &tmp).await.c(d!("rename"))?; - run_cmd( - "qemu-img", - &[ - "convert", - "-f", - "raw", - "-O", - "qcow2", - &tmp.display().to_string(), - &target.display().to_string(), - ], - ) - .await?; - tokio::fs::remove_file(&tmp).await.ok(); - } - - println!("[image] {name} ready: {}", human_size(&target).await); - Ok(()) -} - -// ── Jail images (FreeBSD) ─────────────────────────────────────────── - -async fn create_jail(name: &str, image_dir: &Path) -> Result<()> { - let target = image_dir.join(name); - - if target.exists() { - println!("[image] {name} already exists"); - return Ok(()); - } - - println!("[image] fetching FreeBSD base for jail..."); - tokio::fs::create_dir_all(&target).await.c(d!("mkdir"))?; - - // Detect FreeBSD version - let ver = Command::new("freebsd-version") - .output() - .await - .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) - .unwrap_or_else(|_| "14.3-RELEASE".into()); - - // Extract major version for URL - let major = ver.split('.').next().unwrap_or("14"); - let url = format!("https://download.freebsd.org/releases/amd64/{major}.3-RELEASE/base.txz"); - - let txz = format!("{}/base.txz", target.display()); - download_file(&url, Path::new(&txz)).await?; - - println!("[image] extracting base..."); - run_cmd("tar", &["xf", &txz, "-C", &target.display().to_string()]).await?; - tokio::fs::remove_file(&txz).await.ok(); - - // Configure the jail root - let etc = target.join("etc"); - tokio::fs::write(etc.join("resolv.conf"), "nameserver 8.8.8.8\n") - .await - .ok(); - tokio::fs::write( - etc.join("rc.conf"), - "sendmail_enable=\"NONE\"\nsyslogd_flags=\"-ss\"\n", - ) - .await - .ok(); - - println!("[image] {name} ready: FreeBSD jail base"); - Ok(()) -} - -// ── Public entry point ────────────────────────────────────────────── - -/// Create a specific image by recipe name. -pub async fn create_image(name: &str, image_dir: &Path) -> Result<()> { - let recipe = RECIPES - .iter() - .find(|r| r.name == name) - .ok_or_else(|| eg!("unknown image recipe '{name}' (run 'tt image recipes' to list)"))?; - - match recipe.engine { - "docker" => create_docker(name).await, - "firecracker" => create_firecracker(name, image_dir).await, - "qemu" => create_qemu(name, image_dir).await, - "jail" => create_jail(name, image_dir).await, - _ => Err(eg!("unsupported engine: {}", recipe.engine)), - } -} - -/// Create all images for a given engine. -pub async fn create_all_for_engine(engine: &str, image_dir: &Path) -> Result<()> { - let matching: Vec<_> = RECIPES.iter().filter(|r| r.engine == engine).collect(); - if matching.is_empty() { - return Err(eg!("no recipes for engine '{engine}'")); - } - - for recipe in matching { - println!("\n--- {}: {} ---", recipe.name, recipe.description); - if let Err(e) = create_image(recipe.name, image_dir).await { - eprintln!("[image] WARN: failed to create {}: {e}", recipe.name); - } - } - Ok(()) -} - -/// Create all available images. -pub async fn create_all(image_dir: &Path) -> Result<()> { - for recipe in RECIPES { - // Skip jail on non-FreeBSD and skip FreeBSD-only on Linux - if recipe.engine == "jail" && !cfg!(target_os = "freebsd") { - continue; - } - if recipe.engine == "firecracker" && cfg!(target_os = "freebsd") { - continue; - } - - println!("\n--- {}: {} ---", recipe.name, recipe.description); - if let Err(e) = create_image(recipe.name, image_dir).await { - eprintln!("[image] WARN: failed to create {}: {e}", recipe.name); - } - } - Ok(()) -} - -// ── Helpers ───────────────────────────────────────────────────────── - -async fn download_file(url: &str, dest: &Path) -> Result<()> { - let output = Command::new("curl") - .args(["-fSL", "--progress-bar", "-o"]) - .arg(dest) - .arg(url) - .output() - .await - .c(d!("curl failed"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("download {} failed: {}", url, stderr)); - } - Ok(()) -} - -async fn run_cmd(cmd: &str, args: &[&str]) -> Result<()> { - let output = Command::new(cmd) - .args(args) - .output() - .await - .c(d!(format!("run {cmd}")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("{} failed: {}", cmd, stderr)); - } - Ok(()) -} - -async fn human_size(path: &Path) -> String { - tokio::fs::metadata(path) - .await - .map(|m| { - let bytes = m.len(); - if bytes >= 1024 * 1024 { - format!("{:.1} MB", bytes as f64 / 1024.0 / 1024.0) - } else if bytes >= 1024 { - format!("{:.1} KB", bytes as f64 / 1024.0) - } else { - format!("{bytes} B") - } - }) - .unwrap_or_else(|_| "?".into()) -} - -fn tempdir() -> Result { - let path = PathBuf::from(format!("/tmp/tt-image-{}", std::process::id())); - std::fs::create_dir_all(&path).c(d!("create temp dir"))?; - Ok(path) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn recipes_have_unique_names() { - let mut seen = std::collections::HashSet::new(); - for r in RECIPES { - assert!(seen.insert(r.name), "duplicate recipe: {}", r.name); - } - } - - #[test] - fn docker_tags_resolve() { - assert_eq!(docker_tag("alpine"), "alpine:3.21"); - assert_eq!(docker_tag("debian"), "debian:trixie-slim"); - assert_eq!(docker_tag("unknown"), "unknown"); - } - - #[test] - fn qemu_urls_resolve() { - assert!(qemu_cloud_url("alpine-cloud").is_some()); - assert!(qemu_cloud_url("debian-cloud").is_some()); - assert!(qemu_cloud_url("ubuntu-cloud").is_some()); - assert!(qemu_cloud_url("nonexistent").is_none()); - } - - #[test] - fn all_engines_covered() { - let engines: std::collections::HashSet<&str> = RECIPES.iter().map(|r| r.engine).collect(); - assert!(engines.contains("docker")); - assert!(engines.contains("firecracker")); - assert!(engines.contains("qemu")); - assert!(engines.contains("jail")); - } -} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs deleted file mode 100644 index 7f7530b..0000000 --- a/crates/cli/src/main.rs +++ /dev/null @@ -1,505 +0,0 @@ -//! TTstack CLI — manage your private cloud from the command line. - -mod client; -mod deploy; -mod image_builder; - -use clap::{Parser, Subcommand}; -use client::Client; -use ruc::*; -use ttcore::api::*; -use ttcore::model::*; - -/// TTstack — lightweight private cloud for developers and small teams. -#[derive(Parser)] -#[command(name = "tt", version, about)] -struct Cli { - /// Controller address (overrides ~/.ttconfig). - #[arg(long, short, global = true)] - server: Option, - - /// API key for controller authentication. - #[arg(long, short = 'k', global = true, env = "TT_API_KEY")] - api_key: Option, - - #[command(subcommand)] - cmd: Cmd, -} - -#[derive(Subcommand)] -enum Cmd { - /// Configure the controller address and optional API key. - Config { - /// Controller address, e.g. "10.0.0.1:9200". - addr: String, - /// API key for authentication (optional). - #[arg(long, short = 'k')] - api_key: Option, - }, - /// Show fleet-wide status. - Status, - /// Manage physical hosts. - Host { - #[command(subcommand)] - action: HostCmd, - }, - /// Manage environments. - Env { - #[command(subcommand)] - action: EnvCmd, - }, - /// Manage images. - Image { - #[command(subcommand)] - action: ImageCmd, - }, - /// Deploy TTstack to local or remote hosts. - Deploy { - #[command(subcommand)] - action: DeployCmd, - }, -} - -#[derive(Subcommand)] -enum HostCmd { - /// Register a new host by its agent address. - Add { - /// Agent address, e.g. "10.0.0.2:9100". - addr: String, - }, - /// List all hosts. - List, - /// Show host details. - Show { id: String }, - /// Remove a host from the fleet. - Remove { id: String }, -} - -#[derive(Subcommand)] -enum EnvCmd { - /// Create a new environment with VMs. - Create { - /// Environment name. - name: String, - /// Image name (repeatable). - #[arg(long, short, required = true)] - image: Vec, - /// Engine type: qemu, firecracker, docker (Linux); bhyve, jail (FreeBSD). - #[arg(long, default_value = "qemu")] - engine: String, - /// CPU cores per VM. - #[arg(long)] - cpu: Option, - /// Memory per VM in MiB. - #[arg(long)] - mem: Option, - /// Disk per VM in MiB. - #[arg(long)] - disk: Option, - /// Duplicate each image N times. - #[arg(long, default_value_t = 1)] - dup: u32, - /// Port to expose (repeatable). - #[arg(long, short)] - port: Vec, - /// Environment lifetime in seconds. - #[arg(long)] - lifetime: Option, - /// Block outgoing network traffic from VMs. - #[arg(long)] - deny_outgoing: bool, - /// Owner identifier (defaults to $USER). - #[arg(long)] - owner: Option, - /// SSH public key for VM access (repeatable). Can also be a path to a .pub file. - #[arg(long)] - ssh_key: Vec, - }, - /// List all environments. - List, - /// Show environment details. - Show { name: String }, - /// Delete an environment. - Delete { name: String }, - /// Stop all VMs in an environment. - Stop { name: String }, - /// Start all VMs in an environment. - Start { name: String }, -} - -#[derive(Subcommand)] -enum ImageCmd { - /// List available images across all hosts. - List, - /// List built-in image recipes that can be auto-created. - Recipes, - /// Create an image from a built-in recipe. - Create { - /// Recipe name (see 'tt image recipes'), or "all". - name: String, - /// Image directory (for non-Docker engines). - #[arg(long, default_value = "/home/ttstack/images")] - image_dir: String, - /// Only create images for this engine (docker, firecracker, qemu, jail). - #[arg(long)] - engine: Option, - }, -} - -#[derive(Subcommand)] -enum DeployCmd { - /// Deploy agent on this host (requires root). - Agent { - /// Path to release binaries directory. - #[arg(long, default_value = "./target/release")] - release_dir: String, - }, - /// Deploy controller on this host (requires root). - Ctl { - /// Path to release binaries directory. - #[arg(long, default_value = "./target/release")] - release_dir: String, - }, - /// Deploy both agent and controller on this host (requires root). - All { - /// Path to release binaries directory. - #[arg(long, default_value = "./target/release")] - release_dir: String, - }, - /// Distributed deploy to all hosts defined in a config file. - Dist { - /// Path to deploy config (TOML format). - #[arg(default_value = "deploy.toml")] - config: String, - }, -} - -#[tokio::main] -async fn main() { - let cli = Cli::parse(); - - if let Cmd::Config { addr, api_key } = &cli.cmd { - if let Err(e) = client::save_config(addr, api_key.as_deref()) { - eprintln!("Failed to save config: {e}"); - std::process::exit(1); - } - println!("Controller set to: {addr}"); - if api_key.is_some() { - println!("API key saved."); - } - return; - } - - // Deploy and image-create commands don't need a controller - if let Cmd::Deploy { action } = &cli.cmd { - let result = match action { - DeployCmd::Agent { release_dir } => deploy::deploy_local("agent", release_dir).await, - DeployCmd::Ctl { release_dir } => deploy::deploy_local("ctl", release_dir).await, - DeployCmd::All { release_dir } => deploy::deploy_local("all", release_dir).await, - DeployCmd::Dist { config } => deploy::deploy_distributed(config).await, - }; - if let Err(e) = result { - eprintln!("Error: {e}"); - std::process::exit(1); - } - return; - } - - if let Cmd::Image { - action: ImageCmd::Recipes, - } = &cli.cmd - { - image_builder::list_recipes(); - return; - } - - if let Cmd::Image { - action: - ImageCmd::Create { - name, - image_dir, - engine, - }, - } = &cli.cmd - { - let dir = std::path::Path::new(image_dir); - let result = if name == "all" { - if let Some(eng) = engine { - image_builder::create_all_for_engine(eng, dir).await - } else { - image_builder::create_all(dir).await - } - } else { - image_builder::create_image(name, dir).await - }; - if let Err(e) = result { - eprintln!("Error: {e}"); - std::process::exit(1); - } - return; - } - - let (addr, api_key) = if let Some(server) = cli.server { - (server, cli.api_key) - } else if let Some(cfg) = client::load_config() { - (cfg.addr, cli.api_key.or(cfg.api_key)) - } else { - eprintln!("No controller address. Run: tt config "); - std::process::exit(1); - }; - - let c = Client::new(&addr, api_key.as_deref()); - - let result = match cli.cmd { - Cmd::Config { .. } | Cmd::Deploy { .. } => unreachable!(), - Cmd::Status => cmd_status(&c).await, - Cmd::Host { action } => cmd_host(&c, action).await, - Cmd::Env { action } => cmd_env(&c, action).await, - Cmd::Image { action } => cmd_image(&c, action).await, - }; - - if let Err(e) = result { - eprintln!("Error: {e}"); - std::process::exit(1); - } -} - -// ── Command Implementations ───────────────────────────────────────── - -async fn cmd_status(c: &Client) -> Result<()> { - let s: FleetStatus = c.get("/api/status").await?; - println!("Fleet Status"); - println!(" Hosts: {}/{} online", s.hosts_online, s.hosts); - println!(" VMs: {}", s.total_vms); - println!(" Envs: {}", s.total_envs); - println!(" CPU: {}/{} cores", s.cpu_used, s.cpu_total); - println!(" Memory: {}/{} MB", s.mem_used, s.mem_total); - println!(" Disk: {}/{} MB", s.disk_used, s.disk_total); - Ok(()) -} - -async fn cmd_host(c: &Client, action: HostCmd) -> Result<()> { - match action { - HostCmd::Add { addr } => { - let host: Host = c.post("/api/hosts", &RegisterHostReq { addr }).await?; - println!("Host registered: {} ({})", host.id, host.addr); - println!(" Engines: {:?}", host.engines); - println!(" Storage: {}", host.storage); - println!( - " Resources: {} CPU, {} MB RAM, {} MB disk", - host.resource.cpu_total, host.resource.mem_total, host.resource.disk_total - ); - } - HostCmd::List => { - let hosts: Vec = c.get("/api/hosts").await?; - if hosts.is_empty() { - println!("No hosts registered."); - return Ok(()); - } - println!( - "{:<12} {:<22} {:<8} {:>6} {:>8} {:>8}", - "ID", "ADDR", "STATE", "CPU", "MEM(MB)", "VMs" - ); - for h in hosts { - println!( - "{:<12} {:<22} {:<8} {:>3}/{:<3} {:>4}/{:<4} {:>4}", - h.id, - h.addr, - format!("{:?}", h.state).to_lowercase(), - h.resource.cpu_used, - h.resource.cpu_total, - h.resource.mem_used, - h.resource.mem_total, - h.resource.vm_count, - ); - } - } - HostCmd::Show { id } => { - let h: Host = c.get(&format!("/api/hosts/{id}")).await?; - println!("Host: {}", h.id); - println!(" Address: {}", h.addr); - println!(" State: {:?}", h.state); - println!(" Engines: {:?}", h.engines); - println!(" Storage: {}", h.storage); - println!( - " CPU: {}/{}", - h.resource.cpu_used, h.resource.cpu_total - ); - println!( - " Memory: {}/{} MB", - h.resource.mem_used, h.resource.mem_total - ); - println!( - " Disk: {}/{} MB", - h.resource.disk_used, h.resource.disk_total - ); - println!(" VMs: {}", h.resource.vm_count); - } - HostCmd::Remove { id } => { - c.delete(&format!("/api/hosts/{id}")).await?; - println!("Host removed: {id}"); - } - } - Ok(()) -} - -async fn cmd_env(c: &Client, action: EnvCmd) -> Result<()> { - match action { - EnvCmd::Create { - name, - image, - engine, - cpu, - mem, - disk, - dup, - port, - lifetime, - deny_outgoing, - owner, - ssh_key, - } => { - let engine: Engine = engine - .parse() - .map_err(|e: Box| eg!(e.to_string()))?; - - let owner = owner - .or_else(|| std::env::var("USER").ok()) - .unwrap_or_else(|| "default".to_string()); - - // Resolve SSH keys: if a value looks like a file path, read it - let ssh_keys: Vec = ssh_key - .into_iter() - .map(|k| { - if (k.ends_with(".pub") || k.starts_with('/') || k.starts_with("~/")) - && !k.starts_with("ssh-") - { - let path = if k.starts_with("~/") { - k.replacen("~", &std::env::var("HOME").unwrap_or_default(), 1) - } else { - k.clone() - }; - std::fs::read_to_string(&path) - .map(|s| s.trim().to_string()) - .unwrap_or(k) - } else { - k - } - }) - .collect(); - - let mut vms = Vec::new(); - for img in &image { - for _ in 0..dup { - vms.push(VmSpec { - image: img.clone(), - engine, - cpu, - mem, - disk, - ports: port.clone(), - deny_outgoing, - ssh_keys: vec![], - }); - } - } - - let req = CreateEnvReq { - id: name.clone(), - owner, - vms, - lifetime, - ssh_keys, - }; - - let detail: EnvDetail = c.post("/api/envs", &req).await?; - println!("Environment created: {name}"); - println!(" VMs: {}", detail.vms.len()); - for vm in &detail.vms { - println!( - " {} [{}] {} — {} ports: {:?}", - vm.id, vm.engine, vm.image, vm.ip, vm.port_map - ); - } - for w in &detail.warnings { - eprintln!(" warning: {w}"); - } - } - EnvCmd::List => { - let envs: Vec = c.get("/api/envs").await?; - if envs.is_empty() { - println!("No environments."); - return Ok(()); - } - println!("{:<16} {:<12} {:<8} {:>4}", "NAME", "OWNER", "STATE", "VMs"); - for e in envs { - println!( - "{:<16} {:<12} {:<8} {:>4}", - e.id, - e.owner, - format!("{:?}", e.state).to_lowercase(), - e.vm_ids.len(), - ); - } - } - EnvCmd::Show { name } => { - let detail: EnvDetail = c.get(&format!("/api/envs/{name}")).await?; - println!("Environment: {}", detail.env.id); - println!(" Owner: {}", detail.env.owner); - println!(" State: {:?}", detail.env.state); - println!(" VMs: {}", detail.vms.len()); - println!(); - if !detail.vms.is_empty() { - println!( - " {:<14} {:<12} {:<10} {:<8} {:<16} PORTS", - "ID", "IMAGE", "ENGINE", "STATE", "IP" - ); - for vm in &detail.vms { - let ports: String = vm - .port_map - .iter() - .map(|(g, h)| format!("{h}->{g}")) - .collect::>() - .join(", "); - println!( - " {:<14} {:<12} {:<10} {:<8} {:<16} {}", - vm.id, vm.image, vm.engine, vm.state, vm.ip, ports - ); - } - } - } - EnvCmd::Delete { name } => { - c.delete(&format!("/api/envs/{name}")).await?; - println!("Environment deleted: {name}"); - } - EnvCmd::Stop { name } => { - c.post_action(&format!("/api/envs/{name}/stop")).await?; - println!("Environment stopped: {name}"); - } - EnvCmd::Start { name } => { - c.post_action(&format!("/api/envs/{name}/start")).await?; - println!("Environment started: {name}"); - } - } - Ok(()) -} - -async fn cmd_image(c: &Client, action: ImageCmd) -> Result<()> { - match action { - ImageCmd::List => { - let images: Vec = c.get("/api/images").await?; - if images.is_empty() { - println!("No images available."); - return Ok(()); - } - println!("{:<30} {:<12}", "IMAGE", "HOST"); - for img in images { - println!("{:<30} {:<12}", img.name, img.host_id); - } - } - ImageCmd::Recipes | ImageCmd::Create { .. } => { - unreachable!("handled before controller connection") - } - } - Ok(()) -} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml deleted file mode 100644 index 91b5b40..0000000 --- a/crates/core/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "ttcore" -description = "TTstack core library — models, engine traits, and storage abstractions." -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true - -[dependencies] -serde = { workspace = true } -serde_json = { workspace = true } -ruc = { workspace = true } -uuid = { workspace = true } - -[target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies] -nix = { workspace = true } - -[dev-dependencies] -tempfile = { workspace = true } diff --git a/crates/core/src/api.rs b/crates/core/src/api.rs deleted file mode 100644 index 801863a..0000000 --- a/crates/core/src/api.rs +++ /dev/null @@ -1,221 +0,0 @@ -//! API request and response types shared between all components. - -use crate::model::*; -use serde::{Deserialize, Serialize}; - -// ── Agent API (controller → agent) ───────────────────────────────── - -/// Request to create a VM on an agent. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CreateVmReq { - pub vm_id: String, - pub env_id: String, - pub image: String, - pub engine: Engine, - pub cpu: u32, - pub mem: u32, - pub disk: u32, - pub ports: Vec, - pub deny_outgoing: bool, - /// SSH public keys to inject into the VM for tenant access. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub ssh_keys: Vec, -} - -/// Response from agent after creating a VM. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CreateVmResp { - pub vm: Vm, -} - -/// Information reported by an agent about itself. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AgentInfo { - pub host_id: String, - pub resource: Resource, - pub engines: Vec, - pub storage: Storage, - pub images: Vec, -} - -// ── Controller API (CLI → controller) ────────────────────────────── - -/// Specification for a single VM to be created. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VmSpec { - pub image: String, - #[serde(default = "default_engine")] - pub engine: Engine, - pub cpu: Option, - pub mem: Option, - pub disk: Option, - #[serde(default)] - pub ports: Vec, - #[serde(default)] - pub deny_outgoing: bool, - /// Per-VM SSH keys (merged with env-level keys). - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub ssh_keys: Vec, -} - -fn default_engine() -> Engine { - Engine::Qemu -} - -/// Request to create an environment with one or more VMs. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CreateEnvReq { - pub id: String, - pub owner: String, - pub vms: Vec, - /// Lifetime in seconds; `None` means use server default. - pub lifetime: Option, - /// SSH public keys applied to all VMs in this environment. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub ssh_keys: Vec, -} - -/// Full environment details returned to the CLI. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EnvDetail { - pub env: Env, - pub vms: Vec, - /// Non-fatal warnings (e.g. VMs that failed to create). - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub warnings: Vec, -} - -/// Host registration request from CLI or auto-discovery. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RegisterHostReq { - pub addr: String, -} - -/// Summary of available images across the fleet. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ImageInfo { - pub name: String, - pub host_id: String, -} - -/// Global status of the fleet. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct FleetStatus { - pub hosts: u32, - pub hosts_online: u32, - pub total_vms: u32, - pub total_envs: u32, - pub cpu_total: u32, - pub cpu_used: u32, - pub mem_total: u32, - pub mem_used: u32, - pub disk_total: u32, - pub disk_used: u32, -} - -// ── Generic API Wrapper ──────────────────────────────────────────── - -/// Standard API response envelope. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(bound( - serialize = "T: Serialize", - deserialize = "T: serde::de::DeserializeOwned" -))] -pub struct ApiResp { - pub ok: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub data: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -impl ApiResp { - pub fn success(data: T) -> Self { - Self { - ok: true, - data: Some(data), - error: None, - } - } - - pub fn err(msg: impl Into) -> Self { - Self { - ok: false, - data: None, - error: Some(msg.into()), - } - } -} - -/// Convenience for responses with no payload. -pub type ApiRespEmpty = ApiResp<()>; - -impl ApiRespEmpty { - pub fn ok() -> Self { - Self { - ok: true, - data: None, - error: None, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn api_resp_success_roundtrip() { - let resp = ApiResp::success("hello".to_string()); - assert!(resp.ok); - assert_eq!(resp.data.as_deref(), Some("hello")); - assert!(resp.error.is_none()); - - let json = serde_json::to_string(&resp).unwrap(); - let back: ApiResp = serde_json::from_str(&json).unwrap(); - assert!(back.ok); - assert_eq!(back.data, resp.data); - } - - #[test] - fn api_resp_error() { - let resp = ApiResp::::err("boom"); - assert!(!resp.ok); - assert!(resp.data.is_none()); - assert_eq!(resp.error.as_deref(), Some("boom")); - } - - #[test] - fn api_resp_empty_ok() { - let resp = ApiRespEmpty::ok(); - assert!(resp.ok); - assert!(resp.data.is_none()); - assert!(resp.error.is_none()); - - let json = serde_json::to_string(&resp).unwrap(); - assert!(json.contains(r#""ok":true"#)); - assert!(!json.contains("data")); - assert!(!json.contains("error")); - } - - #[test] - fn api_resp_skip_none_fields() { - let resp = ApiResp::success(42u32); - let json = serde_json::to_string(&resp).unwrap(); - assert!(!json.contains("error")); - - let resp = ApiResp::::err("fail"); - let json = serde_json::to_string(&resp).unwrap(); - assert!(!json.contains("data")); - } - - #[test] - fn vm_spec_defaults() { - let json = r#"{"image": "ubuntu"}"#; - let spec: VmSpec = serde_json::from_str(json).unwrap(); - assert_eq!(spec.image, "ubuntu"); - assert_eq!(spec.engine, Engine::Qemu); // default - assert!(spec.ports.is_empty()); - assert!(!spec.deny_outgoing); - } -} diff --git a/crates/core/src/auth.rs b/crates/core/src/auth.rs deleted file mode 100644 index 54d1fe3..0000000 --- a/crates/core/src/auth.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! Shared authentication utilities. - -/// Constant-time string comparison to prevent timing attacks. -/// -/// Both inputs are compared byte-by-byte; the result is only -/// determined after all bytes have been examined, preventing an -/// attacker from learning the expected key one character at a time. -pub fn constant_time_eq(a: &str, b: &str) -> bool { - let a = a.as_bytes(); - let b = b.as_bytes(); - if a.len() != b.len() { - return false; - } - let mut diff = 0u8; - for (x, y) in a.iter().zip(b.iter()) { - diff |= x ^ y; - } - diff == 0 -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn equal_strings() { - assert!(constant_time_eq("abc", "abc")); - assert!(constant_time_eq("", "")); - assert!(constant_time_eq( - "tt-0123456789abcdef", - "tt-0123456789abcdef" - )); - } - - #[test] - fn different_strings() { - assert!(!constant_time_eq("abc", "abd")); - assert!(!constant_time_eq("abc", "ab")); - assert!(!constant_time_eq("", "a")); - } -} diff --git a/crates/core/src/engine/bhyve.rs b/crates/core/src/engine/bhyve.rs deleted file mode 100644 index 2109c30..0000000 --- a/crates/core/src/engine/bhyve.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! Bhyve engine implementation (FreeBSD only). -//! -//! Bhyve is the native hypervisor on FreeBSD. This module is only -//! compiled on FreeBSD targets via `#[cfg(target_os = "freebsd")]`. - -use super::VmEngine; -use crate::model::{RUN_DIR, Vm, VmState}; -use crate::net; -use ruc::*; -use std::process::Command; - -pub struct BhyveEngine; - -impl BhyveEngine { - pub fn new() -> Self { - Self - } - - fn pid_path(vm: &Vm) -> String { - format!("{RUN_DIR}/bhyve-{}.pid", vm.id) - } - - fn read_pid(vm: &Vm) -> Option { - std::fs::read_to_string(Self::pid_path(vm)) - .ok() - .and_then(|s| s.trim().parse().ok()) - } -} - -impl VmEngine for BhyveEngine { - fn create( - &self, - vm: &Vm, - image_path: &str, - _disk_format: &str, - _ssh_keys: &[String], - ) -> Result<()> { - // Load the VM into bhyve via bhyveload - let output = Command::new("bhyveload") - .args(["-m", &format!("{}M", vm.mem)]) - .args(["-d", image_path]) - .arg(&vm.id) - .output() - .c(d!("failed to run bhyveload"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("bhyveload failed: {}", stderr)); - } - - // Launch the VM as a background daemon - let tap = net::tap_name(&vm.id); - let pid_path = format!("{RUN_DIR}/bhyve-{}.pid", vm.id); - std::fs::create_dir_all(RUN_DIR).c(d!("create pid dir"))?; - - let child = Command::new("bhyve") - .args(["-A", "-H", "-P"]) - .args(["-c", &vm.cpu.to_string()]) - .args(["-m", &format!("{}M", vm.mem)]) - .args(["-s", "0:0,hostbridge"]) - .args(["-s", &format!("3:0,virtio-blk,{image_path}")]) - .args(["-s", &format!("4:0,virtio-net,{tap}")]) - .args(["-s", "31,lpc"]) - .args(["-l", "com1,/dev/null"]) - .arg(&vm.id) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .spawn() - .c(d!("failed to spawn bhyve"))?; - - // Record PID - std::fs::write(&pid_path, child.id().to_string()).c(d!("write pid"))?; - - Ok(()) - } - - fn start(&self, _vm: &Vm) -> Result<()> { - // Bhyve doesn't support pause/resume natively; - // "start" after destroy requires re-create. - Err(eg!( - "bhyve does not support in-place restart; re-create the VM" - )) - } - - fn stop(&self, vm: &Vm) -> Result<()> { - // Kill the bhyve process first, then clean up the VM device - if let Some(pid) = Self::read_pid(vm) { - let _ = nix::sys::signal::kill( - nix::unistd::Pid::from_raw(pid), - nix::sys::signal::Signal::SIGTERM, - ); - // Give it a moment to exit gracefully - std::thread::sleep(std::time::Duration::from_millis(500)); - } - - let output = Command::new("bhyvectl") - .args(["--destroy", "--vm", &vm.id]) - .output() - .c(d!("bhyvectl destroy"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - // Not fatal — the VM device may already be gone - eprintln!( - "[bhyve] WARN: bhyvectl --destroy failed for {}: {}", - vm.id, stderr - ); - } - - Ok(()) - } - - fn destroy(&self, vm: &Vm) -> Result<()> { - // Kill the bhyve process if still alive - if let Some(pid) = Self::read_pid(vm) { - let _ = nix::sys::signal::kill( - nix::unistd::Pid::from_raw(pid), - nix::sys::signal::Signal::SIGKILL, - ); - } - - // Clean up the bhyve VM device - let _ = Command::new("bhyvectl") - .args(["--destroy", "--vm", &vm.id]) - .output(); - - let _ = std::fs::remove_file(Self::pid_path(vm)); - - Ok(()) - } - - fn state(&self, vm: &Vm) -> Result { - let output = Command::new("bhyvectl") - .args(["--get-lowmem", "--vm", &vm.id]) - .output(); - - match output { - Ok(o) if o.status.success() => Ok(VmState::Running), - _ => Ok(VmState::Stopped), - } - } - - fn name(&self) -> &'static str { - "bhyve" - } -} diff --git a/crates/core/src/engine/docker.rs b/crates/core/src/engine/docker.rs deleted file mode 100644 index 2531063..0000000 --- a/crates/core/src/engine/docker.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! Docker / Podman container engine implementation. -//! -//! Auto-detects whether `docker` or `podman` is available and uses -//! whichever is found (preferring podman for rootless operation). - -use super::VmEngine; -use crate::model::{Vm, VmState}; -use ruc::*; -use std::process::Command; -use std::sync::LazyLock; - -/// Cached path to the container runtime binary. -/// -/// Prefer `docker` when available since it has wider ecosystem -/// compatibility; fall back to `podman` for rootless operation. -static RUNTIME: LazyLock<&'static str> = LazyLock::new(|| { - if Command::new("docker") - .arg("--version") - .output() - .map(|o| o.status.success()) - .unwrap_or(false) - { - "docker" - } else { - "podman" - } -}); - -pub struct DockerEngine; - -impl Default for DockerEngine { - fn default() -> Self { - Self::new() - } -} - -impl DockerEngine { - pub fn new() -> Self { - Self - } - - fn runtime() -> &'static str { - &RUNTIME - } - - /// Container name derived from VM id. - fn container_name(vm: &Vm) -> String { - format!("tt-{}", vm.id) - } -} - -impl VmEngine for DockerEngine { - fn create( - &self, - vm: &Vm, - _image_path: &str, - _disk_format: &str, - _ssh_keys: &[String], - ) -> Result<()> { - let name = Self::container_name(vm); - let rt = Self::runtime(); - - let mut cmd = Command::new(rt); - cmd.args(["run", "-d", "--name", &name]) - .args(["--cpus", &vm.cpu.to_string()]) - .args(["--memory", &format!("{}m", vm.mem)]); - - // Publish port mappings - for (&guest, &host) in &vm.port_map { - cmd.args(["-p", &format!("{host}:{guest}")]); - } - - // The image name is used directly as the container image reference - cmd.arg(&vm.image); - - let output = cmd.output().c(d!("failed to spawn container"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("{} run failed: {}", rt, stderr)); - } - - Ok(()) - } - - fn start(&self, vm: &Vm) -> Result<()> { - let name = Self::container_name(vm); - let output = Command::new(Self::runtime()) - .args(["start", &name]) - .output() - .c(d!())?; - - if !output.status.success() { - return Err(eg!("container start failed")); - } - Ok(()) - } - - fn stop(&self, vm: &Vm) -> Result<()> { - let name = Self::container_name(vm); - let output = Command::new(Self::runtime()) - .args(["stop", "-t", "10", &name]) - .output() - .c(d!())?; - - if !output.status.success() { - return Err(eg!("container stop failed")); - } - Ok(()) - } - - fn destroy(&self, vm: &Vm) -> Result<()> { - let name = Self::container_name(vm); - // Force remove the container - let output = Command::new(Self::runtime()) - .args(["rm", "-f", &name]) - .output() - .c(d!())?; - - if !output.status.success() { - return Err(eg!("container remove failed")); - } - Ok(()) - } - - fn state(&self, vm: &Vm) -> Result { - let name = Self::container_name(vm); - let output = Command::new(Self::runtime()) - .args(["inspect", "-f", "{{.State.Status}}", &name]) - .output() - .c(d!())?; - - if !output.status.success() { - return Ok(VmState::Stopped); - } - - let status = String::from_utf8_lossy(&output.stdout); - match status.trim() { - "running" => Ok(VmState::Running), - "paused" => Ok(VmState::Paused), - "exited" | "dead" | "created" => Ok(VmState::Stopped), - _ => Ok(VmState::Failed), - } - } - - fn name(&self) -> &'static str { - "docker" - } -} diff --git a/crates/core/src/engine/firecracker.rs b/crates/core/src/engine/firecracker.rs deleted file mode 100644 index 38811c9..0000000 --- a/crates/core/src/engine/firecracker.rs +++ /dev/null @@ -1,204 +0,0 @@ -//! Firecracker microVM engine implementation. -//! -//! Uses the Firecracker VMM for lightweight, fast-booting microVMs. -//! Communicates with the Firecracker process via its REST API socket. - -use super::VmEngine; -use crate::model::{RUN_DIR, Vm, VmState}; -use ruc::*; -use std::path::Path; -use std::process::Command; - -pub struct FirecrackerEngine; - -impl Default for FirecrackerEngine { - fn default() -> Self { - Self::new() - } -} - -impl FirecrackerEngine { - pub fn new() -> Self { - Self - } - - fn socket_path(vm: &Vm) -> String { - format!("{RUN_DIR}/fc-{}.sock", vm.id) - } - - fn pid_path(vm: &Vm) -> String { - format!("{RUN_DIR}/fc-{}.pid", vm.id) - } - - fn config_path(vm: &Vm) -> String { - format!("{RUN_DIR}/fc-{}.json", vm.id) - } - - fn read_pid(vm: &Vm) -> Result { - let path = Self::pid_path(vm); - let content = std::fs::read_to_string(&path).c(d!("read fc pid"))?; - content.trim().parse::().c(d!("invalid pid")) - } - - fn write_config(&self, vm: &Vm, image_path: &str) -> Result<()> { - let tap = crate::net::tap_name(&vm.id); - let config = serde_json::json!({ - "boot-source": { - "kernel_image_path": format!("{image_path}/vmlinux"), - "boot_args": "console=ttyS0 reboot=k panic=1 pci=off" - }, - "drives": [{ - "drive_id": "rootfs", - "path_on_host": format!("{image_path}/rootfs.ext4"), - "is_root_device": true, - "is_read_only": false - }], - "machine-config": { - "vcpu_count": vm.cpu, - "mem_size_mib": vm.mem, - }, - "network-interfaces": [{ - "iface_id": "eth0", - "host_dev_name": tap, - }] - }); - - let path = Self::config_path(vm); - let json = serde_json::to_string_pretty(&config).c(d!("serialize config"))?; - std::fs::write(&path, json).c(d!("write config")) - } -} - -impl VmEngine for FirecrackerEngine { - fn create( - &self, - vm: &Vm, - image_path: &str, - _disk_format: &str, - _ssh_keys: &[String], - ) -> Result<()> { - std::fs::create_dir_all(RUN_DIR).c(d!("create runtime dir"))?; - - self.write_config(vm, image_path)?; - - let sock = Self::socket_path(vm); - let config = Self::config_path(vm); - - let mut child = Command::new("firecracker") - .args(["--api-sock", &sock]) - .args(["--config-file", &config]) - .spawn() - .c(d!("spawn firecracker"))?; - - let pid = child.id(); - std::fs::write(Self::pid_path(vm), pid.to_string()).c(d!("write pid"))?; - - // Spawn a reaper thread so the child process is wait()ed on, - // preventing zombie processes if the Firecracker VM exits. - std::thread::spawn(move || { - let _ = child.wait(); - }); - - Ok(()) - } - - fn start(&self, vm: &Vm) -> Result<()> { - let sock = Self::socket_path(vm); - if !Path::new(&sock).exists() { - return Err(eg!("firecracker socket not found for VM {}", vm.id)); - } - - // Resume a paused VM via the Firecracker API - let output = Command::new("curl") - .args([ - "--unix-socket", - &sock, - "-s", - "-X", - "PATCH", - "http://localhost/vm", - "-H", - "Content-Type: application/json", - "-d", - r#"{"state": "Resumed"}"#, - ]) - .output() - .c(d!("resume firecracker"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("firecracker resume failed: {}", stderr)); - } - Ok(()) - } - - fn stop(&self, vm: &Vm) -> Result<()> { - let sock = Self::socket_path(vm); - if !Path::new(&sock).exists() { - return Ok(()); - } - - // Pause the VM via the Firecracker API (preserves the process) - let output = Command::new("curl") - .args([ - "--unix-socket", - &sock, - "-s", - "-X", - "PATCH", - "http://localhost/vm", - "-H", - "Content-Type: application/json", - "-d", - r#"{"state": "Paused"}"#, - ]) - .output() - .c(d!("pause firecracker"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("firecracker pause failed: {}", stderr)); - } - Ok(()) - } - - fn destroy(&self, vm: &Vm) -> Result<()> { - if let Ok(pid) = Self::read_pid(vm) { - let _ = nix::sys::signal::kill( - nix::unistd::Pid::from_raw(pid as i32), - nix::sys::signal::Signal::SIGKILL, - ); - } - - let _ = std::fs::remove_file(Self::socket_path(vm)); - let _ = std::fs::remove_file(Self::pid_path(vm)); - let _ = std::fs::remove_file(Self::config_path(vm)); - - Ok(()) - } - - fn state(&self, vm: &Vm) -> Result { - match Self::read_pid(vm) { - Ok(pid) if Path::new(&format!("/proc/{pid}")).exists() => { - // Query Firecracker API to distinguish Running vs Paused - let sock = Self::socket_path(vm); - if Path::new(&sock).exists() - && let Ok(output) = Command::new("curl") - .args(["--unix-socket", &sock, "-s", "http://localhost/vm"]) - .output() - { - let body = String::from_utf8_lossy(&output.stdout); - if body.contains("\"Paused\"") { - return Ok(VmState::Paused); - } - } - Ok(VmState::Running) - } - _ => Ok(VmState::Stopped), - } - } - - fn name(&self) -> &'static str { - "firecracker" - } -} diff --git a/crates/core/src/engine/jail.rs b/crates/core/src/engine/jail.rs deleted file mode 100644 index df6eab2..0000000 --- a/crates/core/src/engine/jail.rs +++ /dev/null @@ -1,147 +0,0 @@ -//! FreeBSD Jail engine implementation. -//! -//! Uses FreeBSD jails for lightweight OS-level virtualization. -//! Each jail gets its own root filesystem, network stack, and process space. - -use super::VmEngine; -use crate::model::{Vm, VmState}; -use ruc::*; -use std::path::Path; -use std::process::Command; - -pub struct JailEngine; - -impl Default for JailEngine { - fn default() -> Self { - Self::new() - } -} - -impl JailEngine { - pub fn new() -> Self { - Self - } - - /// Jail name derived from VM id. - fn jail_name(vm: &Vm) -> String { - format!("tt-{}", vm.id) - } -} - -impl VmEngine for JailEngine { - fn create( - &self, - vm: &Vm, - image_path: &str, - _disk_format: &str, - ssh_keys: &[String], - ) -> Result<()> { - let name = Self::jail_name(vm); - - // jail requires an absolute path for mount.devfs - let abs_path = Path::new(image_path) - .canonicalize() - .c(d!("canonicalize jail path"))?; - - // Inject SSH keys into the jail rootfs before starting - if !ssh_keys.is_empty() { - let ssh_dir = abs_path.join("root/.ssh"); - std::fs::create_dir_all(&ssh_dir).c(d!("create .ssh dir in jail"))?; - let ak = ssh_keys.join("\n") + "\n"; - std::fs::write(ssh_dir.join("authorized_keys"), ak) - .c(d!("write authorized_keys in jail"))?; - // Ensure correct permissions (readable only by owner) - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&ssh_dir, std::fs::Permissions::from_mode(0o700)) - .unwrap_or(()); - std::fs::set_permissions( - ssh_dir.join("authorized_keys"), - std::fs::Permissions::from_mode(0o600), - ) - .unwrap_or(()); - } - } - - // Create the jail with the given root filesystem - let output = Command::new("jail") - .args(["-c"]) - .arg(format!("name={name}")) - .arg(format!("path={}", abs_path.display())) - .arg("host.hostname=ttstack") - .arg(format!("ip4.addr={}", vm.ip)) - .arg("persist") - .arg("mount.devfs") - .output() - .c(d!("create jail"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("jail create failed: {}", stderr)); - } - - Ok(()) - } - - fn start(&self, vm: &Vm) -> Result<()> { - // For jails that were stopped with 'persist' flag, we re-create - let name = Self::jail_name(vm); - - let output = Command::new("jail") - .args(["-m"]) - .arg(format!("name={name}")) - .arg("persist") - .output() - .c(d!("start jail"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("jail start failed: {}", stderr)); - } - - Ok(()) - } - - fn stop(&self, vm: &Vm) -> Result<()> { - let name = Self::jail_name(vm); - - // Kill all processes then remove the jail - let output = Command::new("jail") - .args(["-r", &name]) - .output() - .c(d!("stop jail"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("jail stop failed: {}", stderr)); - } - - Ok(()) - } - - fn destroy(&self, vm: &Vm) -> Result<()> { - // Stop first, then the image cleanup is handled by the runtime - let _ = self.stop(vm); - Ok(()) - } - - fn state(&self, vm: &Vm) -> Result { - let name = Self::jail_name(vm); - - let output = Command::new("jls") - .args(["-j", &name, "jid"]) - .output() - .c(d!("query jail"))?; - - if output.status.success() { - Ok(VmState::Running) - } else { - Ok(VmState::Stopped) - } - } - - fn name(&self) -> &'static str { - "jail" - } -} diff --git a/crates/core/src/engine/mod.rs b/crates/core/src/engine/mod.rs deleted file mode 100644 index 40f7c6c..0000000 --- a/crates/core/src/engine/mod.rs +++ /dev/null @@ -1,71 +0,0 @@ -//! VM / container engine abstraction. -//! -//! Each engine implements [`VmEngine`] to provide a uniform interface -//! for creating, starting, stopping, and destroying instances. -//! -//! Platform-specific engines: -//! - **Linux**: Qemu, Firecracker, Docker/Podman -//! - **FreeBSD**: Bhyve, Jail - -#[cfg(target_os = "freebsd")] -pub mod bhyve; -pub mod docker; -#[cfg(target_os = "linux")] -pub mod firecracker; -#[cfg(target_os = "freebsd")] -pub mod jail; -#[cfg(target_os = "linux")] -pub mod qemu; - -use crate::model::{Engine, Vm, VmState}; -use ruc::*; - -/// Trait implemented by each hypervisor / container engine. -pub trait VmEngine: Send + Sync { - /// Create and boot a new VM from the given disk path. - /// - /// - `disk_format`: image format (`"qcow2"` for file-based, `"raw"` for zvol). - /// - `ssh_keys`: public keys to inject for tenant SSH access. - fn create( - &self, - vm: &Vm, - image_path: &str, - disk_format: &str, - ssh_keys: &[String], - ) -> Result<()>; - - /// Start a previously stopped VM. - fn start(&self, vm: &Vm) -> Result<()>; - - /// Gracefully stop a running VM. - fn stop(&self, vm: &Vm) -> Result<()>; - - /// Destroy the VM and clean up all associated resources. - fn destroy(&self, vm: &Vm) -> Result<()>; - - /// Query the current state of the VM. - fn state(&self, vm: &Vm) -> Result; - - /// Human-readable engine name. - fn name(&self) -> &'static str; -} - -/// Create an engine instance for the given [`Engine`] kind. -/// -/// # Panics -/// Panics if the requested engine is not available on the current platform. -pub fn create_engine(kind: Engine) -> Box { - match kind { - #[cfg(target_os = "linux")] - Engine::Qemu => Box::new(qemu::QemuEngine::new()), - #[cfg(target_os = "linux")] - Engine::Firecracker => Box::new(firecracker::FirecrackerEngine::new()), - Engine::Docker => Box::new(docker::DockerEngine::new()), - #[cfg(target_os = "freebsd")] - Engine::Bhyve => Box::new(bhyve::BhyveEngine::new()), - #[cfg(target_os = "freebsd")] - Engine::Jail => Box::new(jail::JailEngine::new()), - #[allow(unreachable_patterns)] - other => panic!("engine {other} is not supported on this platform"), - } -} diff --git a/crates/core/src/engine/qemu.rs b/crates/core/src/engine/qemu.rs deleted file mode 100644 index fcce6aa..0000000 --- a/crates/core/src/engine/qemu.rs +++ /dev/null @@ -1,331 +0,0 @@ -//! QEMU/KVM engine implementation. -//! -//! Launches VMs via `qemu-system-x86_64` with KVM acceleration. -//! Each VM gets its own tap device connected to the host bridge. - -use super::VmEngine; -use crate::model::{RUN_DIR, Vm, VmState}; -use ruc::*; -use std::path::Path; -use std::process::Command; - -pub struct QemuEngine; - -impl Default for QemuEngine { - fn default() -> Self { - Self::new() - } -} - -impl QemuEngine { - pub fn new() -> Self { - Self - } - - fn build_cmd(&self, vm: &Vm, disk_path: &str, disk_format: &str) -> Command { - let tap = crate::net::tap_name(&vm.id); - let mut cmd = Command::new("qemu-system-x86_64"); - cmd.args(["-enable-kvm", "-daemonize"]) - .args(["-name", &vm.id]) - .args(["-m", &format!("{}M", vm.mem)]) - .args(["-smp", &vm.cpu.to_string()]) - .args([ - "-drive", - &format!("file={disk_path},format={disk_format},if=virtio"), - ]) - .args([ - "-netdev", - &format!("tap,id=net0,ifname={tap},script=no,downscript=no"), - ]) - .args(["-device", "virtio-net-pci,netdev=net0"]) - .args(["-pidfile", &self.pid_path(vm)]) - .args([ - "-monitor", - &format!("unix:{},server,nowait", self.monitor_path(vm)), - ]) - .args(["-vnc", "none"]); - - // Attach cloud-init seed ISO if it exists (for cloud images) - let seed = self.seed_path(vm); - if Path::new(&seed).exists() { - cmd.args([ - "-drive", - &format!("file={seed},format=raw,if=virtio,readonly=on"), - ]); - } - - cmd - } - - /// Generate a cloud-init NoCloud seed ISO for the VM. - /// - /// This allows cloud images (Alpine, Debian, Ubuntu) to auto-configure - /// on first boot: configure networking, inject SSH keys, enable sshd. - fn generate_seed_iso(&self, vm: &Vm, ssh_keys: &[String]) -> Result<()> { - let seed_dir = format!("{RUN_DIR}/seed-{}", vm.id); - std::fs::create_dir_all(&seed_dir).c(d!("create seed dir"))?; - - // meta-data - let meta_data = format!("instance-id: {}\nlocal-hostname: {}\n", vm.id, vm.id); - std::fs::write(format!("{seed_dir}/meta-data"), meta_data).c(d!("write meta-data"))?; - - // network-config (v2) — static IP on the virtio NIC - let network_config = format!( - r#"version: 2 -ethernets: - id0: - match: - driver: virtio_net - addresses: - - {ip}/16 - routes: - - to: 0.0.0.0/0 - via: 10.10.0.1 - nameservers: - addresses: - - 8.8.8.8 - - 1.1.1.1 -"#, - ip = vm.ip, - ); - std::fs::write(format!("{seed_dir}/network-config"), network_config) - .c(d!("write network-config"))?; - - // user-data — inject SSH keys, disable password login - let mut user_data = String::from( - "#cloud-config\n\ - disable_root: false\n\ - ssh_pwauth: false\n", - ); - - if !ssh_keys.is_empty() { - user_data.push_str("ssh_authorized_keys:\n"); - for key in ssh_keys { - user_data.push_str(&format!(" - {key}\n")); - } - } - - user_data.push_str( - "runcmd:\n \ - - sed -i 's/^#*PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config\n \ - - sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config\n \ - - systemctl restart sshd 2>/dev/null || service sshd restart 2>/dev/null || rc-service sshd restart 2>/dev/null || true\n", - ); - - std::fs::write(format!("{seed_dir}/user-data"), user_data).c(d!("write user-data"))?; - - // Generate ISO using genisoimage or mkisofs - let seed_iso = self.seed_path(vm); - let meta = format!("{seed_dir}/meta-data"); - let user = format!("{seed_dir}/user-data"); - let netcfg = format!("{seed_dir}/network-config"); - - let output = if Path::new("/usr/bin/genisoimage").exists() { - Command::new("genisoimage") - .args([ - "-output", &seed_iso, "-volid", "cidata", "-joliet", "-rock", "-quiet", - ]) - .args([&meta, &user, &netcfg]) - .output() - .c(d!("generate seed ISO"))? - } else { - Command::new("mkisofs") - .args(["-o", &seed_iso, "-V", "cidata", "-J", "-R", "-quiet"]) - .args([&meta, &user, &netcfg]) - .output() - .c(d!("generate seed ISO"))? - }; - - // Clean up temp dir - let _ = std::fs::remove_dir_all(&seed_dir); - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("seed ISO creation failed: {}", stderr)); - } - - Ok(()) - } - - fn pid_path(&self, vm: &Vm) -> String { - format!("{RUN_DIR}/qemu-{}.pid", vm.id) - } - - fn monitor_path(&self, vm: &Vm) -> String { - format!("{RUN_DIR}/qemu-{}.sock", vm.id) - } - - fn seed_path(&self, vm: &Vm) -> String { - format!("{RUN_DIR}/seed-{}.iso", vm.id) - } - - fn read_pid(&self, vm: &Vm) -> Result { - let path = self.pid_path(vm); - let content = std::fs::read_to_string(&path).c(d!("read pid file"))?; - content.trim().parse::().c(d!("invalid pid")) - } - - fn process_alive(pid: u32) -> bool { - Path::new(&format!("/proc/{pid}")).exists() - } -} - -impl VmEngine for QemuEngine { - fn create( - &self, - vm: &Vm, - image_path: &str, - disk_format: &str, - ssh_keys: &[String], - ) -> Result<()> { - std::fs::create_dir_all(RUN_DIR).c(d!("create runtime dir"))?; - - // Generate cloud-init seed ISO (best-effort; non-cloud images ignore it) - if let Err(e) = self.generate_seed_iso(vm, ssh_keys) { - eprintln!( - "[qemu] WARN: could not create seed ISO for {}: {e} (cloud-init may not work)", - vm.id - ); - } - - let output = self - .build_cmd(vm, image_path, disk_format) - .output() - .c(d!("spawn qemu"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("qemu launch failed: {}", stderr)); - } - - Ok(()) - } - - fn start(&self, vm: &Vm) -> Result<()> { - let sock = self.monitor_path(vm); - if !Path::new(&sock).exists() { - // Monitor socket gone means QEMU exited; re-create the VM. - return Err(eg!( - "VM {} has no monitor socket; it must be re-created", - vm.id - )); - } - - // Send "cont" command to the QEMU monitor to resume execution - let output = Command::new("sh") - .args([ - "-c", - &format!(r#"echo "cont" | socat - UNIX-CONNECT:{sock}"#), - ]) - .output() - .c(d!("qemu monitor cont"))?; - - if !output.status.success() { - return Err(eg!("failed to resume VM via QEMU monitor")); - } - Ok(()) - } - - fn stop(&self, vm: &Vm) -> Result<()> { - // Pause the VM via QEMU monitor "stop" command. - // This freezes the vCPU without killing the QEMU process, - // allowing later resume via "cont". - let sock = self.monitor_path(vm); - if Path::new(&sock).exists() { - let _ = Command::new("sh") - .args([ - "-c", - &format!(r#"echo "stop" | socat - UNIX-CONNECT:{sock}"#), - ]) - .output(); - } - Ok(()) - } - - fn destroy(&self, vm: &Vm) -> Result<()> { - if let Ok(pid) = self.read_pid(vm) - && Self::process_alive(pid) - { - let _ = nix::sys::signal::kill( - nix::unistd::Pid::from_raw(pid as i32), - nix::sys::signal::Signal::SIGKILL, - ); - } - - let _ = std::fs::remove_file(self.pid_path(vm)); - let _ = std::fs::remove_file(self.monitor_path(vm)); - let _ = std::fs::remove_file(self.seed_path(vm)); - - Ok(()) - } - - fn state(&self, vm: &Vm) -> Result { - match self.read_pid(vm) { - Ok(pid) if Self::process_alive(pid) => { - // Query QEMU monitor to distinguish Running vs Paused - let sock = self.monitor_path(vm); - if Path::new(&sock).exists() - && let Ok(output) = Command::new("sh") - .args([ - "-c", - &format!(r#"echo "info status" | socat - UNIX-CONNECT:{sock}"#), - ]) - .output() - { - let body = String::from_utf8_lossy(&output.stdout); - if body.contains("paused") { - return Ok(VmState::Paused); - } - } - Ok(VmState::Running) - } - _ => Ok(VmState::Stopped), - } - } - - fn name(&self) -> &'static str { - "qemu" - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn build_cmd_uses_disk_format() { - // Smoke test: ensure disk_format ends up in the -drive arg - let eng = QemuEngine::new(); - let vm = Vm { - id: "test-vm".into(), - env_id: "e1".into(), - host_id: "h1".into(), - image: "img".into(), - engine: crate::model::Engine::Qemu, - cpu: 2, - mem: 1024, - disk: 10240, - ip: "10.10.0.2".into(), - port_map: Default::default(), - state: VmState::Creating, - created_at: 0, - }; - - let cmd = eng.build_cmd(&vm, "/dev/zvol/tank/clone-1", "raw"); - let args: Vec<_> = cmd - .get_args() - .map(|a| a.to_string_lossy().into_owned()) - .collect(); - let drive_arg = args.iter().find(|a| a.starts_with("file=")).unwrap(); - assert!(drive_arg.contains("format=raw")); - assert!(drive_arg.contains("/dev/zvol/tank/clone-1")); - - let cmd2 = eng.build_cmd(&vm, "/tmp/disk.qcow2", "qcow2"); - let args2: Vec<_> = cmd2 - .get_args() - .map(|a| a.to_string_lossy().into_owned()) - .collect(); - let drive_arg2 = args2.iter().find(|a| a.starts_with("file=")).unwrap(); - assert!(drive_arg2.contains("format=qcow2")); - } -} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs deleted file mode 100644 index f1c5f3e..0000000 --- a/crates/core/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -//! TTstack core library. -//! -//! Provides shared types, engine abstractions, storage backends, and -//! network utilities used by both the host agent and central controller. -//! -//! The [`api`] and [`model`] modules are platform-independent and used -//! by all components (CLI, controller, agent). -//! -//! The [`engine`], [`net`], and [`storage`] modules are only available -//! on Linux and FreeBSD where the agent daemon runs. - -pub mod api; -pub mod auth; -pub mod model; - -pub mod engine; -pub mod net; -pub mod storage; diff --git a/crates/core/src/model.rs b/crates/core/src/model.rs deleted file mode 100644 index 3eb2de1..0000000 --- a/crates/core/src/model.rs +++ /dev/null @@ -1,430 +0,0 @@ -//! Data models for TTstack. -//! -//! All persistent types are serde-serializable for use with vsdb storage -//! and JSON API communication. - -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; -use std::fmt; - -// ── Engine & Backend Enums ────────────────────────────────────────── - -/// Supported hypervisor / container engines. -/// -/// Platform availability: -/// - **Linux**: Qemu, Firecracker, Docker -/// - **FreeBSD**: Bhyve, Jail -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum Engine { - Qemu, - Firecracker, - Bhyve, - Docker, - Jail, -} - -impl fmt::Display for Engine { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Qemu => write!(f, "qemu"), - Self::Firecracker => write!(f, "firecracker"), - Self::Bhyve => write!(f, "bhyve"), - Self::Docker => write!(f, "docker"), - Self::Jail => write!(f, "jail"), - } - } -} - -impl std::str::FromStr for Engine { - type Err = Box; - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "qemu" | "kvm" => Ok(Self::Qemu), - "firecracker" | "fc" => Ok(Self::Firecracker), - "bhyve" => Ok(Self::Bhyve), - "docker" | "podman" => Ok(Self::Docker), - "jail" => Ok(Self::Jail), - _ => Err(format!("unknown engine: {s}").into()), - } - } -} - -/// Storage backend for VM / container images. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum Storage { - /// Plain qcow2 file copies — works on any filesystem. - File, - /// ZFS zvol — raw block devices backed by ZFS volumes. - Zvol, -} - -impl fmt::Display for Storage { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::File => write!(f, "file"), - Self::Zvol => write!(f, "zvol"), - } - } -} - -impl std::str::FromStr for Storage { - type Err = Box; - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "file" => Ok(Self::File), - "zvol" => Ok(Self::Zvol), - _ => Err(format!("unknown storage backend: {s}").into()), - } - } -} - -// ── State Enums ───────────────────────────────────────────────────── - -/// Runtime state of a VM or container. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum VmState { - Running, - Stopped, - Paused, - Creating, - Failed, -} - -impl fmt::Display for VmState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Running => write!(f, "running"), - Self::Stopped => write!(f, "stopped"), - Self::Paused => write!(f, "paused"), - Self::Creating => write!(f, "creating"), - Self::Failed => write!(f, "failed"), - } - } -} - -/// State of an environment (group of VMs). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum EnvState { - Active, - Stopped, -} - -/// Online status of a physical host. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum HostState { - Online, - Offline, -} - -// ── Resource Tracking ─────────────────────────────────────────────── - -/// Aggregated resource information for a host. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct Resource { - pub cpu_total: u32, - pub cpu_used: u32, - /// Total memory in MiB. - pub mem_total: u32, - /// Used memory in MiB. - pub mem_used: u32, - /// Total disk in MiB. - pub disk_total: u32, - /// Used disk in MiB. - pub disk_used: u32, - /// Number of active VMs / containers. - pub vm_count: u32, -} - -impl Resource { - pub fn cpu_free(&self) -> u32 { - self.cpu_total.saturating_sub(self.cpu_used) - } - - pub fn mem_free(&self) -> u32 { - self.mem_total.saturating_sub(self.mem_used) - } - - pub fn disk_free(&self) -> u32 { - self.disk_total.saturating_sub(self.disk_used) - } - - /// Check whether the host can accommodate the given requirement. - pub fn can_fit(&self, cpu: u32, mem: u32, disk: u32) -> bool { - self.cpu_free() >= cpu && self.mem_free() >= mem && self.disk_free() >= disk - } -} - -// ── Core Entities ─────────────────────────────────────────────────── - -/// A physical host in the fleet. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Host { - pub id: String, - /// Agent listen address, e.g. "10.0.0.1:9100". - pub addr: String, - pub resource: Resource, - pub state: HostState, - /// Engines available on this host. - pub engines: Vec, - /// Storage backend used on this host. - pub storage: Storage, - pub registered_at: u64, -} - -/// A VM or container instance managed by an agent. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Vm { - pub id: String, - pub env_id: String, - pub host_id: String, - pub image: String, - pub engine: Engine, - /// Number of vCPUs. - pub cpu: u32, - /// Memory in MiB. - pub mem: u32, - /// Disk in MiB. - pub disk: u32, - /// Internal IP (on the host bridge). - pub ip: String, - /// guest_port → host_port mapping. - pub port_map: BTreeMap, - pub state: VmState, - pub created_at: u64, -} - -/// An environment — a logical group of related VMs. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Env { - pub id: String, - pub owner: String, - pub vm_ids: Vec, - pub created_at: u64, - /// Unix timestamp after which the env auto-expires (0 = never). - pub expires_at: u64, - pub state: EnvState, -} - -// ── Default VM Sizing ─────────────────────────────────────────────── - -/// Default number of vCPUs per VM. -pub const VM_CPU_DEFAULT: u32 = 2; -/// Default memory per VM in MiB (1 GiB). -pub const VM_MEM_DEFAULT: u32 = 1024; -/// Default disk per VM in MiB (40 GiB). -pub const VM_DISK_DEFAULT: u32 = 40 * 1024; -/// Maximum environment lifetime in seconds (6 hours). -pub const MAX_LIFETIME: u64 = 6 * 3600; -/// Maximum hosts in the fleet. -pub const MAX_HOSTS: usize = 50; -/// Maximum total VM instances across the fleet. -pub const MAX_VMS: usize = 1000; - -/// Directory for engine PID files, sockets, and other runtime state. -pub const RUN_DIR: &str = "/home/ttstack/run"; - -// ── Input Validation ──────────────────────────────────────────────── - -/// Validate that a name (env, host, image) is safe. -/// -/// Rejects path traversal (`..`), shell metacharacters, and excessive length. -pub fn validate_name(name: &str, label: &str) -> std::result::Result<(), String> { - if name.is_empty() { - return Err(format!("{label} cannot be empty")); - } - if name.len() > 128 { - return Err(format!("{label} too long (max 128 chars)")); - } - // Only allow alphanumeric, hyphen, underscore, and dot. - // This prevents path traversal, shell injection, and filesystem issues. - if !name - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') - { - return Err(format!( - "{label} contains invalid characters (only a-z, A-Z, 0-9, '-', '_', '.' allowed)" - )); - } - if name.starts_with('.') || name.contains("..") { - return Err(format!("{label} must not start with '.' or contain '..'")); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - // ── Engine ────────────────────────────────────────────────────── - - #[test] - fn engine_display_roundtrip() { - for e in [ - Engine::Qemu, - Engine::Firecracker, - Engine::Bhyve, - Engine::Docker, - Engine::Jail, - ] { - let s = e.to_string(); - let parsed: Engine = s.parse().unwrap(); - assert_eq!(e, parsed); - } - } - - #[test] - fn engine_aliases() { - assert_eq!("kvm".parse::().unwrap(), Engine::Qemu); - assert_eq!("fc".parse::().unwrap(), Engine::Firecracker); - assert_eq!("podman".parse::().unwrap(), Engine::Docker); - assert_eq!("QEMU".parse::().unwrap(), Engine::Qemu); - } - - #[test] - fn engine_unknown() { - assert!("foobar".parse::().is_err()); - } - - #[test] - fn engine_serde_json() { - let e = Engine::Firecracker; - let json = serde_json::to_string(&e).unwrap(); - assert_eq!(json, r#""firecracker""#); - let back: Engine = serde_json::from_str(&json).unwrap(); - assert_eq!(back, e); - } - - // ── Storage ───────────────────────────────────────────────────── - - #[test] - fn storage_display_roundtrip() { - for s in [Storage::File, Storage::Zvol] { - let text = s.to_string(); - let parsed: Storage = text.parse().unwrap(); - assert_eq!(s, parsed); - } - } - - #[test] - fn storage_unknown() { - assert!("ntfs".parse::().is_err()); - } - - // ── VmState ───────────────────────────────────────────────────── - - #[test] - fn vmstate_display() { - assert_eq!(VmState::Running.to_string(), "running"); - assert_eq!(VmState::Stopped.to_string(), "stopped"); - assert_eq!(VmState::Paused.to_string(), "paused"); - assert_eq!(VmState::Creating.to_string(), "creating"); - assert_eq!(VmState::Failed.to_string(), "failed"); - } - - // ── Resource ──────────────────────────────────────────────────── - - #[test] - fn resource_free_values() { - let r = Resource { - cpu_total: 16, - cpu_used: 6, - mem_total: 32768, - mem_used: 8192, - disk_total: 500_000, - disk_used: 100_000, - vm_count: 3, - }; - assert_eq!(r.cpu_free(), 10); - assert_eq!(r.mem_free(), 24576); - assert_eq!(r.disk_free(), 400_000); - } - - #[test] - fn resource_free_saturates() { - let r = Resource { - cpu_total: 4, - cpu_used: 10, // over-committed - ..Default::default() - }; - assert_eq!(r.cpu_free(), 0); // saturates, no panic - } - - #[test] - fn resource_can_fit() { - let r = Resource { - cpu_total: 8, - cpu_used: 4, - mem_total: 16384, - mem_used: 8192, - disk_total: 200_000, - disk_used: 100_000, - vm_count: 2, - }; - assert!(r.can_fit(4, 8192, 100_000)); // exact fit - assert!(r.can_fit(1, 1, 1)); // plenty of room - assert!(!r.can_fit(5, 1, 1)); // cpu insufficient - assert!(!r.can_fit(1, 9000, 1)); // mem insufficient - assert!(!r.can_fit(1, 1, 200_000)); // disk insufficient - } - - #[test] - fn resource_default_is_zero() { - let r = Resource::default(); - assert_eq!(r.cpu_total, 0); - assert_eq!(r.vm_count, 0); - assert!(!r.can_fit(1, 1, 1)); - } - - // ── Constants ─────────────────────────────────────────────────── - - #[test] - fn constants_sane() { - assert!(VM_CPU_DEFAULT > 0); - assert!(VM_MEM_DEFAULT > 0); - assert!(VM_DISK_DEFAULT > 0); - assert!(MAX_LIFETIME > 0); - assert!(MAX_HOSTS > 0 && MAX_HOSTS <= 100); - assert!(MAX_VMS > 0 && MAX_VMS <= 10_000); - } - - // ── Validation ────────────────────────────────────────────────── - - #[test] - fn validate_name_ok() { - assert!(validate_name("ubuntu-22.04", "image").is_ok()); - assert!(validate_name("my-env", "env").is_ok()); - assert!(validate_name("a", "x").is_ok()); - } - - #[test] - fn validate_name_rejects_traversal() { - assert!(validate_name("../etc/passwd", "image").is_err()); - assert!(validate_name("foo/../bar", "image").is_err()); - assert!(validate_name("foo/bar", "image").is_err()); - } - - #[test] - fn validate_name_rejects_empty_and_long() { - assert!(validate_name("", "env").is_err()); - let long = "a".repeat(200); - assert!(validate_name(&long, "env").is_err()); - } - - #[test] - fn validate_name_rejects_null() { - assert!(validate_name("foo\0bar", "id").is_err()); - } - - #[test] - fn validate_name_rejects_spaces_and_special() { - assert!(validate_name("bad name", "env").is_err()); - assert!(validate_name("bad!", "env").is_err()); - assert!(validate_name("bad@name", "env").is_err()); - assert!(validate_name(".hidden", "env").is_err()); - } -} diff --git a/crates/core/src/net.rs b/crates/core/src/net.rs deleted file mode 100644 index f68508e..0000000 --- a/crates/core/src/net.rs +++ /dev/null @@ -1,478 +0,0 @@ -//! Network utilities for TTstack. -//! -//! Manages the virtual network infrastructure on each host: -//! - A bridge device for VM connectivity -//! - TAP devices for individual VMs -//! - Firewall NAT rules for port forwarding -//! -//! **Linux**: uses `ip`, `nftables` -//! **FreeBSD**: uses `ifconfig`, `pf` - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -use ruc::*; -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -use std::process::Command; - -/// Default bridge name on each host. -pub const BRIDGE_NAME: &str = "tt0"; -/// Bridge IP address (gateway for VMs). -pub const BRIDGE_ADDR: &str = "10.10.0.1"; -/// Bridge subnet mask. -pub const BRIDGE_CIDR: &str = "10.10.0.1/16"; -/// nftables table name (Linux). -#[cfg(target_os = "linux")] -pub const NFT_TABLE: &str = "tt-nat"; - -/// Derive an IP address for a VM from a sequential index (0..65534). -/// -/// Produces addresses in the 10.10.x.y range, skipping .0 and .255. -pub fn vm_ip(index: u32) -> String { - let index = index + 1; // skip .0.0 - let hi = (index / 254) & 0xFF; - let lo = (index % 254) + 1; - format!("10.10.{hi}.{lo}") -} - -/// TAP device name for a VM. -/// -/// Uses a hash of the VM ID to guarantee uniqueness even for long IDs. -/// Result is always <= 15 chars (IFNAMSIZ). -pub fn tap_name(vm_id: &str) -> String { - use std::collections::hash_map::DefaultHasher; - use std::hash::{Hash, Hasher}; - let mut h = DefaultHasher::new(); - vm_id.hash(&mut h); - let hash = h.finish(); - // "tt-" + 12 hex chars = 15 chars exactly - format!("tt-{:012x}", hash & 0xFFFF_FFFF_FFFF) -} - -// ═══════════════════════════════════════════════════════════════════ -// Linux implementation -// ═══════════════════════════════════════════════════════════════════ - -#[cfg(target_os = "linux")] -mod platform { - use super::*; - - pub fn setup_bridge() -> Result<()> { - if bridge_exists()? { - return Ok(()); - } - - run(&["ip", "link", "add", BRIDGE_NAME, "type", "bridge"])?; - run(&["ip", "addr", "add", BRIDGE_CIDR, "dev", BRIDGE_NAME])?; - run(&["ip", "link", "set", BRIDGE_NAME, "up"])?; - - // Enable IP forwarding - std::fs::write("/proc/sys/net/ipv4/ip_forward", "1").c(d!("enable ip_forward"))?; - - Ok(()) - } - - pub fn bridge_exists() -> Result { - let output = Command::new("ip") - .args(["link", "show", BRIDGE_NAME]) - .output() - .c(d!())?; - Ok(output.status.success()) - } - - pub fn create_tap(vm_id: &str) -> Result<()> { - let tap = tap_name(vm_id); - - run(&["ip", "tuntap", "add", "dev", &tap, "mode", "tap"])?; - run(&["ip", "link", "set", &tap, "master", BRIDGE_NAME])?; - run(&["ip", "link", "set", &tap, "up"])?; - - Ok(()) - } - - pub fn destroy_tap(vm_id: &str) -> Result<()> { - let tap = tap_name(vm_id); - let _ = run(&["ip", "link", "del", &tap]); - Ok(()) - } - - pub fn setup_nat() -> Result<()> { - nft(&format!("add table ip {NFT_TABLE}"))?; - - nft(&format!( - "add chain ip {NFT_TABLE} prerouting {{ type nat hook prerouting priority -100; policy accept; }}" - ))?; - - nft(&format!( - "add chain ip {NFT_TABLE} postrouting {{ type nat hook postrouting priority 100; policy accept; }}" - ))?; - - // Flush both chains on startup to avoid duplicate/stale rules. - // Per-VM port forwards in prerouting will be restored from the - // database by the agent's recovery loop. - let _ = nft(&format!("flush chain ip {NFT_TABLE} postrouting")); - let _ = nft(&format!("flush chain ip {NFT_TABLE} prerouting")); - - nft(&format!( - "add rule ip {NFT_TABLE} postrouting ip saddr 10.10.0.0/16 masquerade" - ))?; - - Ok(()) - } - - pub fn add_port_forward(host_port: u16, vm_ip_addr: &str, guest_port: u16) -> Result<()> { - nft(&format!( - "add rule ip {NFT_TABLE} prerouting tcp dport {host_port} dnat to {vm_ip_addr}:{guest_port}" - )) - } - - pub fn remove_port_forwards(vm_ip_addr: &str) -> Result<()> { - let output = Command::new("nft") - .args(["-a", "list", "chain", "ip", NFT_TABLE, "prerouting"]) - .output() - .c(d!())?; - - if !output.status.success() { - return Ok(()); - } - - let listing = String::from_utf8_lossy(&output.stdout); - for line in listing.lines() { - if line.contains(vm_ip_addr) - && let Some(handle) = line - .rsplit("handle ") - .next() - .and_then(|h| h.trim().parse::().ok()) - { - let _ = nft(&format!( - "delete rule ip {NFT_TABLE} prerouting handle {handle}" - )); - } - } - - Ok(()) - } - - pub fn deny_outgoing(vm_ip_addr: &str) -> Result<()> { - let _ = nft(&format!( - "add set ip {NFT_TABLE} denylist {{ type ipv4_addr; }}" - )); - let _ = nft(&format!( - "add chain ip {NFT_TABLE} forward {{ type filter hook forward priority 0; policy accept; }}" - )); - let _ = nft(&format!( - "add rule ip {NFT_TABLE} forward ip saddr @denylist drop" - )); - - nft(&format!( - "add element ip {NFT_TABLE} denylist {{ {vm_ip_addr} }}" - )) - } - - pub fn allow_outgoing(vm_ip_addr: &str) -> Result<()> { - let _ = nft(&format!( - "delete element ip {NFT_TABLE} denylist {{ {vm_ip_addr} }}" - )); - Ok(()) - } - - fn nft(rule: &str) -> Result<()> { - use std::io::Write; - let mut child = Command::new("nft") - .arg("-f") - .arg("-") - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .c(d!("nft spawn"))?; - - if let Some(mut stdin) = child.stdin.take() { - let _ = stdin.write_all(rule.as_bytes()); - let _ = stdin.write_all(b"\n"); - } - - let output = child.wait_with_output().c(d!("nft wait"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("nft {}: {}", rule, stderr)); - } - - Ok(()) - } - - fn run(args: &[&str]) -> Result<()> { - let output = Command::new(args[0]) - .args(&args[1..]) - .output() - .c(d!(args.join(" ")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("{}: {}", args.join(" "), stderr)); - } - - Ok(()) - } -} - -// ═══════════════════════════════════════════════════════════════════ -// FreeBSD implementation -// ═══════════════════════════════════════════════════════════════════ - -#[cfg(target_os = "freebsd")] -mod platform { - use super::*; - - pub fn setup_bridge() -> Result<()> { - if bridge_exists()? { - return Ok(()); - } - - run(&["ifconfig", "bridge", "create", "name", BRIDGE_NAME])?; - run(&["ifconfig", BRIDGE_NAME, "inet", BRIDGE_CIDR])?; - run(&["ifconfig", BRIDGE_NAME, "up"])?; - - // Enable IP forwarding - run(&["sysctl", "net.inet.ip.forwarding=1"])?; - - Ok(()) - } - - pub fn bridge_exists() -> Result { - let output = Command::new("ifconfig").arg(BRIDGE_NAME).output().c(d!())?; - Ok(output.status.success()) - } - - pub fn create_tap(vm_id: &str) -> Result<()> { - let tap = tap_name(vm_id); - - run(&["ifconfig", "tap", "create", "name", &tap])?; - run(&["ifconfig", BRIDGE_NAME, "addm", &tap])?; - run(&["ifconfig", &tap, "up"])?; - - Ok(()) - } - - pub fn destroy_tap(vm_id: &str) -> Result<()> { - let tap = tap_name(vm_id); - let _ = run(&["ifconfig", &tap, "destroy"]); - Ok(()) - } - - pub fn setup_nat() -> Result<()> { - // PF should be configured in /etc/pf.conf - // We only enable it here - let _ = run(&["pfctl", "-e"]); - Ok(()) - } - - pub fn add_port_forward(host_port: u16, vm_ip_addr: &str, guest_port: u16) -> Result<()> { - // Add a PF rdr rule via pfctl - let rule = format!( - "rdr pass on egress proto tcp from any to any port {host_port} -> {vm_ip_addr} port {guest_port}" - ); - let output = Command::new("sh") - .args(["-c", &format!(r#"echo '{rule}' | pfctl -a ttstack -f -"#)]) - .output() - .c(d!("pfctl rdr"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("pfctl rdr failed: {}", stderr)); - } - Ok(()) - } - - pub fn remove_port_forwards(vm_ip_addr: &str) -> Result<()> { - // List current rules and remove only those matching this VM's IP - let output = Command::new("pfctl") - .args(["-a", "ttstack", "-s", "rules"]) - .output(); - - if let Ok(output) = output { - let rules = String::from_utf8_lossy(&output.stdout); - let remaining: Vec<&str> = rules - .lines() - .filter(|line| !line.contains(vm_ip_addr)) - .collect(); - - if remaining.is_empty() { - // No rules left — flush the anchor - let _ = run(&["pfctl", "-a", "ttstack", "-F", "rules"]); - } else { - // Reload only the remaining rules - let new_rules = remaining.join("\n"); - let _ = Command::new("sh") - .args([ - "-c", - &format!(r#"echo '{}' | pfctl -a ttstack -f -"#, new_rules), - ]) - .output(); - } - } - - Ok(()) - } - - pub fn deny_outgoing(vm_ip_addr: &str) -> Result<()> { - let rule = format!("block out quick on egress from {vm_ip_addr} to any"); - let output = Command::new("sh") - .args([ - "-c", - &format!(r#"echo '{rule}' | pfctl -a ttstack/deny -f -"#), - ]) - .output() - .c(d!("pfctl deny"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("pfctl deny failed: {}", stderr)); - } - Ok(()) - } - - pub fn allow_outgoing(vm_ip_addr: &str) -> Result<()> { - // List current deny rules and remove only those matching this VM's IP - let output = Command::new("pfctl") - .args(["-a", "ttstack/deny", "-s", "rules"]) - .output(); - - if let Ok(output) = output { - let rules = String::from_utf8_lossy(&output.stdout); - let remaining: Vec<&str> = rules - .lines() - .filter(|line| !line.contains(vm_ip_addr)) - .collect(); - - if remaining.is_empty() { - let _ = run(&["pfctl", "-a", "ttstack/deny", "-F", "rules"]); - } else { - let new_rules = remaining.join("\n"); - let _ = Command::new("sh") - .args([ - "-c", - &format!(r#"echo '{}' | pfctl -a ttstack/deny -f -"#, new_rules), - ]) - .output(); - } - } - - Ok(()) - } - - fn run(args: &[&str]) -> Result<()> { - let output = Command::new(args[0]) - .args(&args[1..]) - .output() - .c(d!(args.join(" ")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("{}: {}", args.join(" "), stderr)); - } - - Ok(()) - } -} - -// ═══════════════════════════════════════════════════════════════════ -// Public re-exports (dispatches to platform module) -// ═══════════════════════════════════════════════════════════════════ - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn setup_bridge() -> Result<()> { - platform::setup_bridge() -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn setup_nat() -> Result<()> { - platform::setup_nat() -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn create_tap(vm_id: &str, _vm_ip_addr: &str) -> Result<()> { - platform::create_tap(vm_id) -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn destroy_tap(vm_id: &str) -> Result<()> { - platform::destroy_tap(vm_id) -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn add_port_forward(host_port: u16, vm_ip_addr: &str, guest_port: u16) -> Result<()> { - platform::add_port_forward(host_port, vm_ip_addr, guest_port) -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn remove_port_forwards(vm_ip_addr: &str) -> Result<()> { - platform::remove_port_forwards(vm_ip_addr) -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn deny_outgoing(vm_ip_addr: &str) -> Result<()> { - platform::deny_outgoing(vm_ip_addr) -} - -#[cfg(any(target_os = "linux", target_os = "freebsd"))] -pub fn allow_outgoing(vm_ip_addr: &str) -> Result<()> { - platform::allow_outgoing(vm_ip_addr) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn vm_ip_first() { - // index=0 → internal=1 → hi=0, lo=2 → 10.10.0.2 - assert_eq!(vm_ip(0), "10.10.0.2"); - } - - #[test] - fn vm_ip_sequential() { - assert_eq!(vm_ip(1), "10.10.0.3"); - // index=252 → internal=253 → hi=0, lo=254 → 10.10.0.254 - assert_eq!(vm_ip(252), "10.10.0.254"); - } - - #[test] - fn vm_ip_wraps_to_next_octet() { - // index=253 → internal=254 → hi=1, lo=254%254+1=1 → 10.10.1.1 - assert_eq!(vm_ip(253), "10.10.1.1"); - } - - #[test] - fn vm_ip_unique_and_valid() { - use std::collections::HashSet; - let mut seen = HashSet::new(); - for i in 0..1000 { - let ip = vm_ip(i); - assert!(seen.insert(ip.clone()), "duplicate IP at index {i}: {ip}"); - // Verify no .0 or .255 in last octet - let lo: u32 = ip.rsplit('.').next().unwrap().parse().unwrap(); - assert!(lo >= 1 && lo <= 254, "invalid lo octet {lo} at index {i}"); - } - } - - #[test] - fn tap_name_fits_ifnamsiz() { - assert!(tap_name("abc").len() <= 15); - assert!(tap_name("a".repeat(200).as_str()).len() <= 15); - } - - #[test] - fn tap_name_deterministic() { - assert_eq!(tap_name("vm1"), tap_name("vm1")); - } - - #[test] - fn tap_name_unique_for_different_ids() { - assert_ne!(tap_name("vm1"), tap_name("vm2")); - // Long IDs that used to collide via truncation are now unique - assert_ne!( - tap_name("very_long_vm_name_1"), - tap_name("very_long_vm_name_2") - ); - } -} diff --git a/crates/core/src/storage/file.rs b/crates/core/src/storage/file.rs deleted file mode 100644 index 1fc01d4..0000000 --- a/crates/core/src/storage/file.rs +++ /dev/null @@ -1,232 +0,0 @@ -//! File-based storage backend. -//! -//! Uses plain file/directory copies for image provisioning. Works on -//! any filesystem. On Linux with CoW filesystems, `cp --reflink=auto` -//! makes copies near-instant. - -use super::ImageStore; -use ruc::*; -use std::path::Path; - -pub struct FileStore; - -impl ImageStore for FileStore { - fn clone_image(&self, base: &str, target: &str) -> Result<()> { - let mut cmd = std::process::Command::new("cp"); - #[cfg(target_os = "linux")] - cmd.args(["--reflink=auto", "-a", base, target]); - #[cfg(not(target_os = "linux"))] - cmd.args(["-a", base, target]); - let output = cmd.output().c(d!("cp image"))?; - - if !output.status.success() { - let err = String::from_utf8_lossy(&output.stderr); - return Err(eg!("image copy failed: {}", err)); - } - - Ok(()) - } - - fn remove_image(&self, path: &str) -> Result<()> { - let p = Path::new(path); - if p.is_dir() { - std::fs::remove_dir_all(p).c(d!("remove dir"))?; - } else if p.exists() { - std::fs::remove_file(p).c(d!("remove file"))?; - } - Ok(()) - } - - fn list_images(&self, base_dir: &str) -> Result> { - let dir = Path::new(base_dir); - if !dir.is_dir() { - return Ok(vec![]); - } - - let mut images = Vec::new(); - for entry in std::fs::read_dir(dir).c(d!("read image dir"))? { - let entry = entry.c(d!("read dir entry"))?; - let name = entry.file_name().to_string_lossy().into_owned(); - if !name.starts_with('.') && !name.starts_with("clone-") { - images.push(name); - } - } - images.sort(); - Ok(images) - } - - fn image_exists(&self, path: &str) -> Result { - Ok(Path::new(path).exists()) - } - - fn resolve_disk(&self, clone_path: &str) -> String { - let p = Path::new(clone_path); - if p.is_dir() { - if let Ok(entries) = std::fs::read_dir(p) { - let files: Vec<_> = entries - .filter_map(|e| e.ok()) - .filter(|e| e.path().is_file()) - .collect(); - // Prefer .qcow2 file - if let Some(qcow2) = files - .iter() - .find(|f| f.path().extension().is_some_and(|ext| ext == "qcow2")) - { - return qcow2.path().to_string_lossy().into_owned(); - } - // Single file — use it directly - if files.len() == 1 { - return files[0].path().to_string_lossy().into_owned(); - } - } - format!("{clone_path}/disk.qcow2") - } else { - clone_path.to_string() - } - } - - fn disk_format(&self) -> &'static str { - "qcow2" - } - - fn name(&self) -> &'static str { - "file" - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn clone_and_remove_file() { - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().join("base.img"); - let clone = dir.path().join("clone.img"); - std::fs::write(&base, b"image-data").unwrap(); - - let store = FileStore; - store - .clone_image(base.to_str().unwrap(), clone.to_str().unwrap()) - .unwrap(); - assert!(clone.exists()); - assert_eq!(std::fs::read(&clone).unwrap(), b"image-data"); - - store.remove_image(clone.to_str().unwrap()).unwrap(); - assert!(!clone.exists()); - } - - #[test] - fn clone_and_remove_directory() { - let dir = tempfile::tempdir().unwrap(); - let base_dir = dir.path().join("base"); - let clone_dir = dir.path().join("clone"); - std::fs::create_dir(&base_dir).unwrap(); - std::fs::write(base_dir.join("disk.qcow2"), b"data").unwrap(); - - let store = FileStore; - store - .clone_image(base_dir.to_str().unwrap(), clone_dir.to_str().unwrap()) - .unwrap(); - assert!(clone_dir.join("disk.qcow2").exists()); - - store.remove_image(clone_dir.to_str().unwrap()).unwrap(); - assert!(!clone_dir.exists()); - } - - #[test] - fn list_images_filters_clones_and_hidden() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("ubuntu"), b"").unwrap(); - std::fs::write(dir.path().join("alpine"), b"").unwrap(); - std::fs::write(dir.path().join(".hidden"), b"").unwrap(); - std::fs::write(dir.path().join("clone-abc"), b"").unwrap(); - - let store = FileStore; - let images = store.list_images(dir.path().to_str().unwrap()).unwrap(); - assert_eq!(images, vec!["alpine", "ubuntu"]); - } - - #[test] - fn list_images_empty_dir() { - let dir = tempfile::tempdir().unwrap(); - let store = FileStore; - let images = store.list_images(dir.path().to_str().unwrap()).unwrap(); - assert!(images.is_empty()); - } - - #[test] - fn list_images_nonexistent_dir() { - let store = FileStore; - let images = store.list_images("/no/such/path").unwrap(); - assert!(images.is_empty()); - } - - #[test] - fn image_exists_check() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("img"); - let store = FileStore; - assert!(!store.image_exists(path.to_str().unwrap()).unwrap()); - std::fs::write(&path, b"").unwrap(); - assert!(store.image_exists(path.to_str().unwrap()).unwrap()); - } - - #[test] - fn remove_nonexistent_is_ok() { - let store = FileStore; - store.remove_image("/no/such/file").unwrap(); - } - - #[test] - fn name_is_file() { - assert_eq!(FileStore.name(), "file"); - } - - #[test] - fn disk_format_is_qcow2() { - assert_eq!(FileStore.disk_format(), "qcow2"); - } - - #[test] - fn resolve_disk_plain_file() { - let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("image.qcow2"); - std::fs::write(&file, b"fake").unwrap(); - let resolved = FileStore.resolve_disk(file.to_str().unwrap()); - assert_eq!(resolved, file.to_str().unwrap()); - } - - #[test] - fn resolve_disk_dir_with_qcow2() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("disk.qcow2"), b"fake").unwrap(); - std::fs::write(dir.path().join("other.txt"), b"other").unwrap(); - let resolved = FileStore.resolve_disk(dir.path().to_str().unwrap()); - assert!(resolved.ends_with("disk.qcow2")); - } - - #[test] - fn resolve_disk_dir_single_file() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("myimage"), b"fake").unwrap(); - let resolved = FileStore.resolve_disk(dir.path().to_str().unwrap()); - assert!(resolved.ends_with("myimage")); - } - - #[test] - fn resolve_disk_dir_fallback() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("a"), b"fake").unwrap(); - std::fs::write(dir.path().join("b"), b"fake").unwrap(); - let resolved = FileStore.resolve_disk(dir.path().to_str().unwrap()); - assert!(resolved.ends_with("disk.qcow2")); - } - - #[test] - fn resolve_disk_empty_dir_fallback() { - let dir = tempfile::tempdir().unwrap(); - let resolved = FileStore.resolve_disk(dir.path().to_str().unwrap()); - assert!(resolved.ends_with("disk.qcow2")); - } -} diff --git a/crates/core/src/storage/mod.rs b/crates/core/src/storage/mod.rs deleted file mode 100644 index 3479525..0000000 --- a/crates/core/src/storage/mod.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! Image storage abstraction. -//! -//! Two backends: plain file copies (`FileStore`) and ZFS zvols (`ZvolStore`). - -pub mod file; -pub mod zvol; - -use crate::model::Storage; -use ruc::*; - -/// Trait for image storage operations. -/// -/// Implementations handle the mechanics of cloning base images into -/// per-VM working copies and cleaning them up on destruction. -pub trait ImageStore: Send + Sync { - /// Clone a base image to a new path for a VM instance. - /// - /// - `base`: path (or dataset name) of the base / template image - /// - `target`: desired path (or dataset name) for the VM's working copy - fn clone_image(&self, base: &str, target: &str) -> Result<()>; - - /// Remove a VM's image clone. - fn remove_image(&self, path: &str) -> Result<()>; - - /// List available base images under the given directory / dataset. - fn list_images(&self, base_dir: &str) -> Result>; - - /// Check whether an image exists at the given path / dataset. - fn image_exists(&self, path: &str) -> Result; - - /// Resolve a clone path to the actual disk path the engine should use. - /// - /// - `FileStore`: searches the directory for a qcow2 file. - /// - `ZvolStore`: returns `/dev/zvol/{dataset}`. - fn resolve_disk(&self, clone_path: &str) -> String; - - /// Disk format string for the engine (e.g. `"qcow2"` or `"raw"`). - fn disk_format(&self) -> &'static str; - - /// Backend name for logging. - fn name(&self) -> &'static str; -} - -/// Create an [`ImageStore`] for the given backend. -pub fn create_store(kind: Storage) -> Box { - match kind { - Storage::File => Box::new(file::FileStore), - Storage::Zvol => Box::new(zvol::ZvolStore), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn create_store_names() { - assert_eq!(create_store(Storage::File).name(), "file"); - assert_eq!(create_store(Storage::Zvol).name(), "zvol"); - } -} diff --git a/crates/core/src/storage/zvol.rs b/crates/core/src/storage/zvol.rs deleted file mode 100644 index d00289b..0000000 --- a/crates/core/src/storage/zvol.rs +++ /dev/null @@ -1,271 +0,0 @@ -//! ZFS zvol storage backend. -//! -//! Uses ZFS volumes (zvols) as raw block devices for VMs. Each base -//! image is a zvol; VM copies are instant clones via snapshots. -//! -//! Advanced features: -//! - **Snapshots**: create, list, destroy, rollback -//! - **zfs send/recv**: full and incremental streams for backup and -//! cross-host migration -//! - **Property queries**: volsize, used, compressratio, etc. - -use super::ImageStore; -use ruc::*; -use std::io::{Read, Write}; -use std::process::{Command, Stdio}; - -/// Fixed snapshot name used for cloning base images. -const CLONE_SNAP: &str = "ttsnap"; - -pub struct ZvolStore; - -// ── Helper: run a zfs command and return stdout or a descriptive error ── - -fn zfs_cmd(args: &[&str]) -> Result { - let output = Command::new("zfs") - .args(args) - .output() - .c(d!("failed to execute zfs"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(eg!("zfs {} failed: {}", args[0], stderr.trim())); - } - - Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) -} - -fn zfs_ok(args: &[&str]) -> bool { - Command::new("zfs") - .args(args) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|s| s.success()) - .unwrap_or(false) -} - -// ── Snapshot management ───────────────────────────────────────────── - -impl ZvolStore { - /// Create a named snapshot of a zvol. - /// - /// Returns the full snapshot name (`dataset@name`). - pub fn create_snapshot(&self, dataset: &str, snap_name: &str) -> Result { - let snap = format!("{dataset}@{snap_name}"); - zfs_cmd(&["snapshot", &snap])?; - Ok(snap) - } - - /// List all snapshots of a dataset, returning full snapshot names. - pub fn list_snapshots(&self, dataset: &str) -> Result> { - let out = zfs_cmd(&["list", "-H", "-o", "name", "-t", "snapshot", "-r", dataset])?; - Ok(out - .lines() - .filter(|l| !l.is_empty()) - .map(String::from) - .collect()) - } - - /// Destroy a snapshot (full name `dataset@snap`). - pub fn destroy_snapshot(&self, snap: &str) -> Result<()> { - zfs_cmd(&["destroy", snap])?; - Ok(()) - } - - /// Rollback a zvol to the given snapshot. - /// - /// **Warning**: destroys all snapshots created after `snap_name`. - pub fn rollback(&self, dataset: &str, snap_name: &str) -> Result<()> { - let snap = format!("{dataset}@{snap_name}"); - zfs_cmd(&["rollback", "-r", &snap])?; - Ok(()) - } - - /// Ensure the clone snapshot (`@ttsnap`) exists on `dataset`. - fn ensure_clone_snap(dataset: &str) -> Result<()> { - let snap = format!("{dataset}@{CLONE_SNAP}"); - if !zfs_ok(&["list", "-t", "snapshot", &snap]) { - zfs_cmd(&["snapshot", &snap])?; - } - Ok(()) - } -} - -// ── zfs send / recv ───────────────────────────────────────────────── - -impl ZvolStore { - /// Full send: write a complete snapshot stream to a writer. - /// - /// `snap` is the full snapshot name, e.g. `tank/images/alpine@backup`. - pub fn send(&self, snap: &str, out: &mut dyn Write) -> Result<()> { - let mut child = Command::new("zfs") - .args(["send", snap]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .c(d!("spawn zfs send"))?; - - if let Some(mut stdout) = child.stdout.take() { - std::io::copy(&mut stdout, out).c(d!("pipe zfs send stream"))?; - } - - let status = child.wait().c(d!("wait zfs send"))?; - if !status.success() { - return Err(eg!("zfs send failed")); - } - Ok(()) - } - - /// Incremental send: write the delta between two snapshots. - /// - /// `from_snap` and `to_snap` are full snapshot names on the same dataset. - pub fn send_incremental( - &self, - from_snap: &str, - to_snap: &str, - out: &mut dyn Write, - ) -> Result<()> { - let mut child = Command::new("zfs") - .args(["send", "-i", from_snap, to_snap]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .c(d!("spawn zfs send -i"))?; - - if let Some(mut stdout) = child.stdout.take() { - std::io::copy(&mut stdout, out).c(d!("pipe incremental stream"))?; - } - - let status = child.wait().c(d!("wait zfs send -i"))?; - if !status.success() { - return Err(eg!("zfs send -i failed")); - } - Ok(()) - } - - /// Receive a zfs stream into a new or existing dataset. - pub fn recv(&self, dataset: &str, input: &mut dyn Read) -> Result<()> { - let mut child = Command::new("zfs") - .args(["recv", dataset]) - .stdin(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .c(d!("spawn zfs recv"))?; - - if let Some(mut stdin) = child.stdin.take() { - std::io::copy(input, &mut stdin).c(d!("pipe to zfs recv"))?; - } - - let status = child.wait().c(d!("wait zfs recv"))?; - if !status.success() { - return Err(eg!("zfs recv failed")); - } - Ok(()) - } - - /// Convenience: full send to a file. - pub fn send_to_file(&self, snap: &str, dest: &str) -> Result<()> { - let mut f = std::fs::File::create(dest).c(d!("create send file"))?; - self.send(snap, &mut f) - } - - /// Convenience: incremental send to a file. - pub fn send_incremental_to_file( - &self, - from_snap: &str, - to_snap: &str, - dest: &str, - ) -> Result<()> { - let mut f = std::fs::File::create(dest).c(d!("create send file"))?; - self.send_incremental(from_snap, to_snap, &mut f) - } - - /// Convenience: recv from a file. - pub fn recv_from_file(&self, dataset: &str, src: &str) -> Result<()> { - let mut f = std::fs::File::open(src).c(d!("open recv file"))?; - self.recv(dataset, &mut f) - } - - /// Build a `zfs send` command for piping to external programs (e.g. ssh). - pub fn send_cmd(&self, snap: &str) -> Command { - let mut cmd = Command::new("zfs"); - cmd.args(["send", snap]); - cmd - } - - /// Build an incremental `zfs send -i` command. - pub fn send_incremental_cmd(&self, from_snap: &str, to_snap: &str) -> Command { - let mut cmd = Command::new("zfs"); - cmd.args(["send", "-i", from_snap, to_snap]); - cmd - } - - /// Build a `zfs recv` command for piping from external programs. - pub fn recv_cmd(&self, dataset: &str) -> Command { - let mut cmd = Command::new("zfs"); - cmd.args(["recv", dataset]); - cmd - } -} - -// ── Property queries ──────────────────────────────────────────────── - -impl ZvolStore { - /// Get a ZFS property value (e.g. `"volsize"`, `"used"`, `"compressratio"`). - pub fn get_property(&self, dataset: &str, prop: &str) -> Result { - zfs_cmd(&["get", "-H", "-o", "value", prop, dataset]) - } - - /// Set a ZFS property. - pub fn set_property(&self, dataset: &str, prop: &str, value: &str) -> Result<()> { - zfs_cmd(&["set", &format!("{prop}={value}"), dataset])?; - Ok(()) - } -} - -// ── ImageStore implementation ─────────────────────────────────────── - -impl ImageStore for ZvolStore { - fn clone_image(&self, base: &str, target: &str) -> Result<()> { - Self::ensure_clone_snap(base)?; - let snap = format!("{base}@{CLONE_SNAP}"); - zfs_cmd(&["clone", &snap, target])?; - Ok(()) - } - - fn remove_image(&self, path: &str) -> Result<()> { - zfs_cmd(&["destroy", "-r", path])?; - Ok(()) - } - - fn list_images(&self, base_dir: &str) -> Result> { - let out = match zfs_cmd(&["list", "-H", "-o", "name", "-r", "-t", "volume", base_dir]) { - Ok(o) => o, - Err(_) => return Ok(vec![]), - }; - - Ok(out - .lines() - .filter(|l| !l.is_empty() && *l != base_dir) - .filter_map(|l| l.rsplit('/').next()) - .map(String::from) - .collect()) - } - - fn image_exists(&self, path: &str) -> Result { - Ok(zfs_ok(&["list", "-H", path])) - } - - fn resolve_disk(&self, clone_path: &str) -> String { - format!("/dev/zvol/{clone_path}") - } - - fn disk_format(&self) -> &'static str { - "raw" - } - - fn name(&self) -> &'static str { - "zvol" - } -} diff --git a/crates/ctl/Cargo.toml b/crates/ctl/Cargo.toml deleted file mode 100644 index 10584f4..0000000 --- a/crates/ctl/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "tt-ctl" -description = "TTstack central controller — fleet management and VM scheduling." -version.workspace = true -edition.workspace = true -authors.workspace = true -license.workspace = true - -[[bin]] -name = "tt-ctl" -path = "src/main.rs" - -[dependencies] -ttcore = { path = "../core" } -serde = { workspace = true } -serde_json = { workspace = true } -ruc = { workspace = true } -rusqlite = { workspace = true } -tokio = { workspace = true } -axum = { workspace = true } -reqwest = { workspace = true } -clap = { workspace = true } -uuid = { workspace = true } diff --git a/crates/ctl/src/auth.rs b/crates/ctl/src/auth.rs deleted file mode 100644 index b4eb94e..0000000 --- a/crates/ctl/src/auth.rs +++ /dev/null @@ -1,47 +0,0 @@ -//! API key authentication middleware. - -use axum::body::Body; -use axum::http::{Request, StatusCode}; -use axum::middleware::Next; -use axum::response::{IntoResponse, Response}; - -/// Create an auth middleware function that validates Bearer tokens. -/// -/// Returns a closure suitable for `axum::middleware::from_fn`. -pub fn make_auth_layer( - expected_key: String, -) -> impl Fn( - Request, - Next, -) -> std::pin::Pin + Send>> -+ Clone -+ Send -+ Sync -+ 'static { - move |req: Request, next: Next| { - let expected = expected_key.clone(); - Box::pin(async move { - let auth_header = req - .headers() - .get("authorization") - .and_then(|v| v.to_str().ok()); - - match auth_header { - Some(value) - if value - .strip_prefix("Bearer ") - .is_some_and(|t| ttcore::auth::constant_time_eq(t, &expected)) => - { - next.run(req).await - } - _ => ( - StatusCode::UNAUTHORIZED, - axum::Json(ttcore::api::ApiResp::<()>::err( - "invalid or missing API key", - )), - ) - .into_response(), - } - }) - } -} diff --git a/crates/ctl/src/config.rs b/crates/ctl/src/config.rs deleted file mode 100644 index 4f07a15..0000000 --- a/crates/ctl/src/config.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! Controller configuration. - -use clap::Parser; - -/// TTstack central controller — fleet management and VM scheduling. -#[derive(Parser, Debug)] -#[command(name = "tt-ctl", version)] -pub struct Config { - /// Listen address for the HTTP API. - #[arg(long, default_value = "0.0.0.0:9200")] - pub listen: String, - - /// Directory for persistent state (SQLite database). - #[arg(long, default_value = "/home/ttstack/ctl")] - pub data_dir: String, - - /// API key for authentication. If set, all API requests must include - /// `Authorization: Bearer `. Can also be provided via TT_API_KEY env var. - #[arg(long, env = "TT_API_KEY")] - pub api_key: Option, -} diff --git a/crates/ctl/src/db.rs b/crates/ctl/src/db.rs deleted file mode 100644 index daedc9a..0000000 --- a/crates/ctl/src/db.rs +++ /dev/null @@ -1,502 +0,0 @@ -//! Persistent state management backed by SQLite. -//! -//! All fleet state (hosts, environments, VMs) is stored in a single -//! SQLite database. Data survives controller restarts. - -use ruc::*; -use rusqlite::Connection; -use ttcore::api::FleetStatus; -use ttcore::model::*; - -/// Current schema version. Bump this when schema changes. -const SCHEMA_VERSION: u32 = 1; - -/// Fleet database — the single source of truth for the controller. -pub struct Db { - conn: Connection, -} - -impl Db { - /// Open or create the database at the given path. - /// - /// Performs automatic schema migration if the existing database - /// has an older version. - pub fn open(path: &str) -> Result { - let conn = Connection::open(path).c(d!("open DB"))?; - - conn.execute_batch( - "PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA foreign_keys=ON;", - ) - .c(d!("set pragmas"))?; - - Self::migrate(&conn)?; - - Ok(Self { conn }) - } - - /// Run schema migrations from the current version to SCHEMA_VERSION. - fn migrate(conn: &Connection) -> Result<()> { - // Create the meta table if it doesn't exist - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS _meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - );", - ) - .c(d!("create meta table"))?; - - let current = Self::get_schema_version(conn)?; - - if current > SCHEMA_VERSION { - return Err(eg!( - "database schema v{} is newer than this binary (v{}); upgrade TTstack first", - current, - SCHEMA_VERSION - )); - } - - if current < 1 { - // v0 → v1: initial schema - conn.execute_batch( - "CREATE TABLE IF NOT EXISTS hosts ( - id TEXT PRIMARY KEY, - data TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS envs ( - id TEXT PRIMARY KEY, - data TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS vms ( - id TEXT PRIMARY KEY, - env_id TEXT NOT NULL, - host_id TEXT NOT NULL, - data TEXT NOT NULL - ); - CREATE INDEX IF NOT EXISTS idx_vms_env ON vms(env_id); - CREATE INDEX IF NOT EXISTS idx_vms_host ON vms(host_id);", - ) - .c(d!("migration v1"))?; - } - - // Future migrations go here: - // if current < 2 { ... } - - Self::set_schema_version(conn, SCHEMA_VERSION)?; - - if current < SCHEMA_VERSION { - eprintln!("database migrated: v{current} → v{SCHEMA_VERSION}"); - } - - Ok(()) - } - - fn get_schema_version(conn: &Connection) -> Result { - let mut stmt = conn - .prepare("SELECT value FROM _meta WHERE key = 'schema_version'") - .c(d!())?; - let mut rows = stmt.query([]).c(d!())?; - match rows.next().c(d!())? { - Some(row) => { - let val: String = row.get(0).c(d!())?; - val.parse::() - .map_err(|_| eg!("invalid schema_version: {val}")) - } - None => Ok(0), // fresh database - } - } - - fn set_schema_version(conn: &Connection, ver: u32) -> Result<()> { - conn.execute( - "INSERT OR REPLACE INTO _meta (key, value) VALUES ('schema_version', ?1)", - rusqlite::params![ver.to_string()], - ) - .c(d!("set schema version"))?; - Ok(()) - } - - // ── Hosts ─────────────────────────────────────────────────────── - - pub fn put_host(&self, host: &Host) -> Result<()> { - let data = serde_json::to_string(host).c(d!("serialize host"))?; - self.conn - .execute( - "INSERT OR REPLACE INTO hosts (id, data) VALUES (?1, ?2)", - rusqlite::params![host.id, data], - ) - .c(d!("put host"))?; - Ok(()) - } - - pub fn get_host(&self, id: &str) -> Result> { - query_one(&self.conn, "SELECT data FROM hosts WHERE id = ?1", [id]) - } - - pub fn remove_host(&self, id: &str) -> Result<()> { - self.conn - .execute("DELETE FROM hosts WHERE id = ?1", [id]) - .c(d!("remove host"))?; - Ok(()) - } - - pub fn list_hosts(&self) -> Result> { - query_all(&self.conn, "SELECT data FROM hosts", []) - } - - pub fn host_count(&self) -> Result { - let count: i64 = self - .conn - .query_row("SELECT COUNT(*) FROM hosts", [], |row| row.get(0)) - .c(d!("count hosts"))?; - Ok(count as usize) - } - - // ── Environments ──────────────────────────────────────────────── - - pub fn put_env(&self, env: &Env) -> Result<()> { - let data = serde_json::to_string(env).c(d!("serialize env"))?; - self.conn - .execute( - "INSERT OR REPLACE INTO envs (id, data) VALUES (?1, ?2)", - rusqlite::params![env.id, data], - ) - .c(d!("put env"))?; - Ok(()) - } - - pub fn get_env(&self, id: &str) -> Result> { - query_one(&self.conn, "SELECT data FROM envs WHERE id = ?1", [id]) - } - - pub fn remove_env(&self, id: &str) -> Result<()> { - self.conn - .execute("DELETE FROM envs WHERE id = ?1", [id]) - .c(d!("remove env"))?; - Ok(()) - } - - pub fn list_envs(&self) -> Result> { - query_all(&self.conn, "SELECT data FROM envs", []) - } - - pub fn env_count(&self) -> Result { - let count: i64 = self - .conn - .query_row("SELECT COUNT(*) FROM envs", [], |row| row.get(0)) - .c(d!("count envs"))?; - Ok(count as usize) - } - - // ── VMs ───────────────────────────────────────────────────────── - - pub fn put_vm(&self, vm: &Vm) -> Result<()> { - let data = serde_json::to_string(vm).c(d!("serialize VM"))?; - self.conn - .execute( - "INSERT OR REPLACE INTO vms (id, env_id, host_id, data) - VALUES (?1, ?2, ?3, ?4)", - rusqlite::params![vm.id, vm.env_id, vm.host_id, data], - ) - .c(d!("put VM"))?; - Ok(()) - } - - pub fn get_vm(&self, id: &str) -> Result> { - query_one(&self.conn, "SELECT data FROM vms WHERE id = ?1", [id]) - } - - pub fn remove_vm(&self, id: &str) -> Result<()> { - self.conn - .execute("DELETE FROM vms WHERE id = ?1", [id]) - .c(d!("remove VM"))?; - Ok(()) - } - - pub fn vm_count(&self) -> Result { - let count: i64 = self - .conn - .query_row("SELECT COUNT(*) FROM vms", [], |row| row.get(0)) - .c(d!("count VMs"))?; - Ok(count as usize) - } - - pub fn vms_by_env(&self, env_id: &str) -> Result> { - query_all( - &self.conn, - "SELECT data FROM vms WHERE env_id = ?1", - [env_id], - ) - } - - pub fn vms_by_host(&self, host_id: &str) -> Result> { - query_all( - &self.conn, - "SELECT data FROM vms WHERE host_id = ?1", - [host_id], - ) - } - - // ── Aggregate Status ──────────────────────────────────────────── - - pub fn fleet_status(&self) -> Result { - let hosts = self.list_hosts()?; - let hosts_online = hosts - .iter() - .filter(|h| h.state == HostState::Online) - .count() as u32; - - let (mut cpu_t, mut cpu_u) = (0u32, 0u32); - let (mut mem_t, mut mem_u) = (0u32, 0u32); - let (mut disk_t, mut disk_u) = (0u32, 0u32); - for h in &hosts { - cpu_t += h.resource.cpu_total; - cpu_u += h.resource.cpu_used; - mem_t += h.resource.mem_total; - mem_u += h.resource.mem_used; - disk_t += h.resource.disk_total; - disk_u += h.resource.disk_used; - } - - Ok(FleetStatus { - hosts: hosts.len() as u32, - hosts_online, - total_vms: self.vm_count()? as u32, - total_envs: self.env_count()? as u32, - cpu_total: cpu_t, - cpu_used: cpu_u, - mem_total: mem_t, - mem_used: mem_u, - disk_total: disk_t, - disk_used: disk_u, - }) - } -} - -// ── Generic Query Helpers ─────────────────────────────────────────── - -fn query_one( - conn: &Connection, - sql: &str, - params: P, -) -> Result> { - let mut stmt = conn.prepare(sql).c(d!("prepare"))?; - let mut rows = stmt.query(params).c(d!("query"))?; - match rows.next().c(d!("next"))? { - Some(row) => { - let data: String = row.get(0).c(d!("get col"))?; - let obj: T = serde_json::from_str(&data).c(d!("deserialize"))?; - Ok(Some(obj)) - } - None => Ok(None), - } -} - -fn query_all( - conn: &Connection, - sql: &str, - params: P, -) -> Result> { - let mut stmt = conn.prepare(sql).c(d!("prepare"))?; - let rows = stmt - .query_map(params, |row| row.get::<_, String>(0)) - .c(d!("query"))?; - let mut result = Vec::new(); - for row in rows { - let data = row.c(d!("read row"))?; - let obj: T = serde_json::from_str(&data).c(d!("deserialize"))?; - result.push(obj); - } - Ok(result) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::BTreeMap; - - fn test_db() -> Db { - Db::open(":memory:").unwrap() - } - - fn make_host(id: &str) -> Host { - Host { - id: id.into(), - addr: format!("{id}:9100"), - resource: Resource { - cpu_total: 8, - mem_total: 16384, - disk_total: 500_000, - ..Default::default() - }, - state: HostState::Online, - engines: vec![Engine::Qemu], - storage: Storage::File, - registered_at: 1000, - } - } - - fn make_env(id: &str) -> Env { - Env { - id: id.into(), - owner: "tester".into(), - vm_ids: vec!["vm1".into()], - created_at: 1000, - expires_at: 2000, - state: EnvState::Active, - } - } - - fn make_vm(id: &str, env_id: &str, host_id: &str) -> Vm { - Vm { - id: id.into(), - env_id: env_id.into(), - host_id: host_id.into(), - image: "ubuntu".into(), - engine: Engine::Qemu, - cpu: 2, - mem: 1024, - disk: 40960, - ip: "10.10.0.1".into(), - port_map: BTreeMap::new(), - state: VmState::Running, - created_at: 1000, - } - } - - // ── Host CRUD ─────────────────────────────────────────────────── - - #[test] - fn host_crud() { - let db = test_db(); - assert_eq!(db.host_count().unwrap(), 0); - - let h = make_host("h1"); - db.put_host(&h).unwrap(); - - assert_eq!(db.host_count().unwrap(), 1); - let got = db.get_host("h1").unwrap().unwrap(); - assert_eq!(got.id, "h1"); - assert_eq!(got.resource.cpu_total, 8); - - db.remove_host("h1").unwrap(); - assert!(db.get_host("h1").unwrap().is_none()); - assert_eq!(db.host_count().unwrap(), 0); - } - - #[test] - fn host_list() { - let db = test_db(); - db.put_host(&make_host("h1")).unwrap(); - db.put_host(&make_host("h2")).unwrap(); - let hosts = db.list_hosts().unwrap(); - assert_eq!(hosts.len(), 2); - } - - #[test] - fn host_upsert() { - let db = test_db(); - let mut h = make_host("h1"); - db.put_host(&h).unwrap(); - h.state = HostState::Offline; - db.put_host(&h).unwrap(); - assert_eq!(db.host_count().unwrap(), 1); - let got = db.get_host("h1").unwrap().unwrap(); - assert_eq!(got.state, HostState::Offline); - } - - // ── Env CRUD ──────────────────────────────────────────────────── - - #[test] - fn env_crud() { - let db = test_db(); - let e = make_env("env1"); - db.put_env(&e).unwrap(); - - assert_eq!(db.env_count().unwrap(), 1); - let got = db.get_env("env1").unwrap().unwrap(); - assert_eq!(got.owner, "tester"); - - db.remove_env("env1").unwrap(); - assert!(db.get_env("env1").unwrap().is_none()); - } - - #[test] - fn env_list() { - let db = test_db(); - db.put_env(&make_env("e1")).unwrap(); - db.put_env(&make_env("e2")).unwrap(); - db.put_env(&make_env("e3")).unwrap(); - assert_eq!(db.list_envs().unwrap().len(), 3); - } - - // ── VM CRUD ───────────────────────────────────────────────────── - - #[test] - fn vm_crud() { - let db = test_db(); - let vm = make_vm("vm1", "env1", "h1"); - db.put_vm(&vm).unwrap(); - - assert_eq!(db.vm_count().unwrap(), 1); - let got = db.get_vm("vm1").unwrap().unwrap(); - assert_eq!(got.image, "ubuntu"); - - db.remove_vm("vm1").unwrap(); - assert!(db.get_vm("vm1").unwrap().is_none()); - } - - #[test] - fn vms_by_env() { - let db = test_db(); - db.put_vm(&make_vm("v1", "env1", "h1")).unwrap(); - db.put_vm(&make_vm("v2", "env1", "h1")).unwrap(); - db.put_vm(&make_vm("v3", "env2", "h1")).unwrap(); - - let vms = db.vms_by_env("env1").unwrap(); - assert_eq!(vms.len(), 2); - - let vms = db.vms_by_env("env2").unwrap(); - assert_eq!(vms.len(), 1); - - let vms = db.vms_by_env("nonexistent").unwrap(); - assert!(vms.is_empty()); - } - - #[test] - fn vms_by_host() { - let db = test_db(); - db.put_vm(&make_vm("v1", "e1", "h1")).unwrap(); - db.put_vm(&make_vm("v2", "e1", "h2")).unwrap(); - - assert_eq!(db.vms_by_host("h1").unwrap().len(), 1); - assert_eq!(db.vms_by_host("h2").unwrap().len(), 1); - assert!(db.vms_by_host("h3").unwrap().is_empty()); - } - - // ── Fleet Status ──────────────────────────────────────────────── - - #[test] - fn fleet_status_aggregates() { - let db = test_db(); - let mut h1 = make_host("h1"); - h1.resource.cpu_used = 2; - h1.resource.mem_used = 4096; - db.put_host(&h1).unwrap(); - - let mut h2 = make_host("h2"); - h2.state = HostState::Offline; - db.put_host(&h2).unwrap(); - - db.put_env(&make_env("e1")).unwrap(); - db.put_vm(&make_vm("v1", "e1", "h1")).unwrap(); - - let status = db.fleet_status().unwrap(); - assert_eq!(status.hosts, 2); - assert_eq!(status.hosts_online, 1); - assert_eq!(status.total_envs, 1); - assert_eq!(status.total_vms, 1); - assert_eq!(status.cpu_total, 16); // 8+8 - assert_eq!(status.cpu_used, 2); - } -} diff --git a/crates/ctl/src/handler.rs b/crates/ctl/src/handler.rs deleted file mode 100644 index c0b8b4f..0000000 --- a/crates/ctl/src/handler.rs +++ /dev/null @@ -1,710 +0,0 @@ -//! HTTP API handlers for the central controller. -//! -//! Handles requests from the CLI and coordinates with host agents. - -use crate::db::Db; -use crate::scheduler; -use axum::Json; -use axum::extract::{Path, State}; -use axum::http::StatusCode; -use axum::response::IntoResponse; -use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, Mutex, MutexGuard}; -use ttcore::api::*; -use ttcore::model::*; - -/// Shared controller state. -pub struct CtlShared { - pub(crate) db: Mutex, - /// API key used for controller→agent communication. - pub api_key: Option, -} - -impl CtlShared { - pub fn new(db: Db, api_key: Option) -> Self { - Self { - db: Mutex::new(db), - api_key, - } - } - - /// Lock the DB mutex, recovering from poisoning. - pub fn lock_db(&self) -> MutexGuard<'_, Db> { - self.db.lock().unwrap_or_else(|e| { - eprintln!("[ctl] WARN: db mutex was poisoned, recovering"); - e.into_inner() - }) - } -} - -pub type CtlState = Arc; - -/// Build an HTTP client for agent communication, with optional Bearer auth. -pub fn agent_client(api_key: Option<&str>, timeout_secs: u64) -> reqwest::Client { - let mut builder = - reqwest::Client::builder().timeout(std::time::Duration::from_secs(timeout_secs)); - if let Some(key) = api_key { - let mut headers = reqwest::header::HeaderMap::new(); - if let Ok(val) = reqwest::header::HeaderValue::from_str(&format!("Bearer {key}")) { - headers.insert(reqwest::header::AUTHORIZATION, val); - } - builder = builder.default_headers(headers); - } - builder.build().unwrap() -} - -// ── Host Management ───────────────────────────────────────────────── - -/// POST /api/hosts — register a new host by its agent address. -pub async fn register_host( - State(db): State, - Json(req): Json, -) -> impl IntoResponse { - let client = agent_client(db.api_key.as_deref(), 30); - let url = format!("http://{}/api/info", req.addr); - - let resp = match client.get(&url).send().await { - Ok(r) => r, - Err(e) => { - return ( - StatusCode::BAD_GATEWAY, - Json(ApiResp::::err(format!( - "cannot reach agent at {}: {e}", - req.addr - ))), - ); - } - }; - - let info: ApiResp = match resp.json().await { - Ok(r) => r, - Err(e) => { - return ( - StatusCode::BAD_GATEWAY, - Json(ApiResp::::err(format!("invalid agent response: {e}"))), - ); - } - }; - - let info = match info.data { - Some(i) => i, - None => { - return ( - StatusCode::BAD_GATEWAY, - Json(ApiResp::::err("agent returned no data")), - ); - } - }; - - let db = db.lock_db(); - - if db.host_count().unwrap_or(0) >= MAX_HOSTS { - return ( - StatusCode::CONFLICT, - Json(ApiResp::::err(format!( - "fleet limit reached ({MAX_HOSTS} hosts)" - ))), - ); - } - - let host = Host { - id: info.host_id, - addr: req.addr, - resource: info.resource, - state: HostState::Online, - engines: info.engines, - storage: info.storage, - registered_at: now(), - }; - - if let Err(e) = db.put_host(&host) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiResp::::err(e.to_string())), - ); - } - - (StatusCode::CREATED, Json(ApiResp::success(host))) -} - -/// GET /api/hosts -pub async fn list_hosts(State(db): State) -> impl IntoResponse { - let db = db.lock_db(); - match db.list_hosts() { - Ok(hosts) => Json(ApiResp::success(hosts)), - Err(e) => Json(ApiResp::>::err(e.to_string())), - } -} - -/// GET /api/hosts/:id -pub async fn get_host(State(db): State, Path(id): Path) -> impl IntoResponse { - let db = db.lock_db(); - match db.get_host(&id) { - Ok(Some(h)) => (StatusCode::OK, Json(ApiResp::success(h))), - Ok(None) => ( - StatusCode::NOT_FOUND, - Json(ApiResp::::err(format!("host not found: {id}"))), - ), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiResp::::err(e.to_string())), - ), - } -} - -/// DELETE /api/hosts/:id -pub async fn remove_host(State(db): State, Path(id): Path) -> impl IntoResponse { - let db = db.lock_db(); - - let vms = db.vms_by_host(&id).unwrap_or_default(); - if !vms.is_empty() { - return ( - StatusCode::CONFLICT, - Json(ApiRespEmpty::err(format!( - "host {id} still has {} VMs; destroy them first", - vms.len() - ))), - ); - } - - if let Err(e) = db.remove_host(&id) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiRespEmpty::err(e.to_string())), - ); - } - - (StatusCode::OK, Json(ApiRespEmpty::ok())) -} - -// ── Environment Management ────────────────────────────────────────── - -/// POST /api/envs — create an environment with VMs. -pub async fn create_env( - State(db): State, - Json(req): Json, -) -> impl IntoResponse { - // Input validation - if let Err(e) = validate_name(&req.id, "env name") { - return (StatusCode::BAD_REQUEST, Json(ApiResp::::err(e))); - } - for spec in &req.vms { - if let Err(e) = validate_name(&spec.image, "image") { - return (StatusCode::BAD_REQUEST, Json(ApiResp::::err(e))); - } - if spec.cpu == Some(0) || spec.mem == Some(0) || spec.disk == Some(0) { - return ( - StatusCode::BAD_REQUEST, - Json(ApiResp::::err( - "cpu, mem, and disk must be > 0 if specified", - )), - ); - } - } - - // Reserve the environment name under lock to prevent races - let hosts = { - let db = db.lock_db(); - - if let Ok(Some(_)) = db.get_env(&req.id) { - return ( - StatusCode::CONFLICT, - Json(ApiResp::::err(format!( - "environment '{}' already exists", - req.id - ))), - ); - } - - // Insert a placeholder env to reserve the name while we create VMs. - // This prevents concurrent requests from creating the same env. - let placeholder = Env { - id: req.id.clone(), - owner: req.owner.clone(), - vm_ids: vec![], - created_at: now(), - expires_at: 0, - state: EnvState::Active, - }; - if let Err(e) = db.put_env(&placeholder) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiResp::::err(e.to_string())), - ); - } - - match db.list_hosts() { - Ok(h) => h, - Err(e) => { - let _ = db.remove_env(&req.id); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiResp::::err(e.to_string())), - ); - } - } - }; - - // Fetch available images from all online hosts for scheduling validation - let client = agent_client(db.api_key.as_deref(), 30); - let host_images = fetch_host_images(&hosts, &client).await; - - let placements = match scheduler::schedule_env(&hosts, &req.vms, &host_images) { - Ok(p) => p, - Err(e) => { - // Clean up the placeholder - let db = db.lock_db(); - let _ = db.remove_env(&req.id); - return ( - StatusCode::UNPROCESSABLE_ENTITY, - Json(ApiResp::::err(e.to_string())), - ); - } - }; - - let created_at = now(); - let expires_at = req - .lifetime - .filter(|<| lt > 0) - .map(|lt| created_at + lt.min(MAX_LIFETIME)) - .unwrap_or(created_at + MAX_LIFETIME); - - let mut vm_ids = Vec::new(); - let mut created_vms = Vec::new(); - let mut warnings = Vec::new(); - - // Create VMs on agents (no lock held during HTTP calls) - for (spec, placement) in &placements { - let vm_id = uuid::Uuid::new_v4().to_string()[..12].to_string(); - let cpu = spec.cpu.unwrap_or(VM_CPU_DEFAULT); - let mem = spec.mem.unwrap_or(VM_MEM_DEFAULT); - let disk = spec.disk.unwrap_or(VM_DISK_DEFAULT); - - let agent_req = CreateVmReq { - vm_id: vm_id.clone(), - env_id: req.id.clone(), - image: spec.image.clone(), - engine: spec.engine, - cpu, - mem, - disk, - ports: spec.ports.clone(), - deny_outgoing: spec.deny_outgoing, - ssh_keys: { - let mut keys = req.ssh_keys.clone(); - keys.extend(spec.ssh_keys.iter().cloned()); - keys.dedup(); - keys - }, - }; - - let url = format!("http://{}/api/vms", placement.host_addr); - match client.post(&url).json(&agent_req).send().await { - Ok(r) if r.status().is_success() => { - if let Ok(body) = r.json::>().await - && let Some(data) = body.data - { - vm_ids.push(vm_id); - created_vms.push(data.vm); - continue; - } - warnings.push(format!("unparseable response from {}", placement.host_addr)); - } - Ok(r) => { - warnings.push(format!( - "agent {} returned {}", - placement.host_addr, - r.status() - )); - } - Err(e) => { - warnings.push(format!("failed to reach {}: {e}", placement.host_addr)); - } - } - } - - if !warnings.is_empty() { - eprintln!( - "[ctl] WARN: partial env '{}' creation: {}/{} VMs failed: {}", - req.id, - warnings.len(), - req.vms.len(), - warnings.join("; ") - ); - } - - if created_vms.is_empty() && !req.vms.is_empty() { - // Clean up the placeholder - let db = db.lock_db(); - let _ = db.remove_env(&req.id); - return ( - StatusCode::BAD_GATEWAY, - Json(ApiResp::::err( - "all VM creation attempts failed; check agent connectivity", - )), - ); - } - - // Update the placeholder with the real environment data - let env = Env { - id: req.id.clone(), - owner: req.owner.clone(), - vm_ids: vm_ids.clone(), - created_at, - expires_at, - state: EnvState::Active, - }; - - { - let db = db.lock_db(); - let _ = db.put_env(&env); - for vm in &created_vms { - let _ = db.put_vm(vm); - } - } - - refresh_all_hosts(&db, &client).await; - - let detail = EnvDetail { - env, - vms: created_vms, - warnings, - }; - - (StatusCode::CREATED, Json(ApiResp::success(detail))) -} - -/// GET /api/envs -pub async fn list_envs(State(db): State) -> impl IntoResponse { - let db = db.lock_db(); - match db.list_envs() { - Ok(envs) => Json(ApiResp::success(envs)), - Err(e) => Json(ApiResp::>::err(e.to_string())), - } -} - -/// GET /api/envs/:id -pub async fn get_env(State(db): State, Path(id): Path) -> impl IntoResponse { - let db = db.lock_db(); - match db.get_env(&id) { - Ok(Some(env)) => { - let vms = db.vms_by_env(&id).unwrap_or_default(); - ( - StatusCode::OK, - Json(ApiResp::success(EnvDetail { - env, - vms, - warnings: vec![], - })), - ) - } - Ok(None) => ( - StatusCode::NOT_FOUND, - Json(ApiResp::::err(format!( - "environment not found: {id}" - ))), - ), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiResp::::err(e.to_string())), - ), - } -} - -/// DELETE /api/envs/:id -pub async fn delete_env(State(db): State, Path(id): Path) -> impl IntoResponse { - let (vms, hosts) = { - let db = db.lock_db(); - match db.get_env(&id) { - Ok(Some(_)) => {} - Ok(None) => { - return ( - StatusCode::NOT_FOUND, - Json(ApiRespEmpty::err(format!("environment not found: {id}"))), - ); - } - Err(e) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiRespEmpty::err(e.to_string())), - ); - } - }; - let vms = db.vms_by_env(&id).unwrap_or_default(); - let hosts = db.list_hosts().unwrap_or_default(); - (vms, hosts) - }; - - let client = agent_client(db.api_key.as_deref(), 30); - for vm in &vms { - if let Some(host) = hosts.iter().find(|h| h.id == vm.host_id) { - let url = format!("http://{}/api/vms/{}", host.addr, vm.id); - match client.delete(&url).send().await { - Ok(r) if !r.status().is_success() => { - eprintln!( - "[ctl] WARN: agent {} returned {} when deleting VM {}", - host.addr, - r.status(), - vm.id - ); - } - Err(e) => { - eprintln!( - "[ctl] WARN: failed to contact agent {} to delete VM {}: {e}", - host.addr, vm.id - ); - } - _ => {} - } - } - } - - { - let db = db.lock_db(); - for vm in &vms { - let _ = db.remove_vm(&vm.id); - } - let _ = db.remove_env(&id); - } - - (StatusCode::OK, Json(ApiRespEmpty::ok())) -} - -/// POST /api/envs/:id/stop -pub async fn stop_env(State(db): State, Path(id): Path) -> impl IntoResponse { - let (mut env, vms, hosts) = { - let db = db.lock_db(); - let env = match db.get_env(&id) { - Ok(Some(e)) => e, - _ => { - return ( - StatusCode::NOT_FOUND, - Json(ApiRespEmpty::err(format!("environment not found: {id}"))), - ); - } - }; - let vms = db.vms_by_env(&id).unwrap_or_default(); - let hosts = db.list_hosts().unwrap_or_default(); - (env, vms, hosts) - }; - - let client = agent_client(db.api_key.as_deref(), 30); - for vm in &vms { - if let Some(host) = hosts.iter().find(|h| h.id == vm.host_id) { - let url = format!("http://{}/api/vms/{}/stop", host.addr, vm.id); - match client.post(&url).send().await { - Ok(r) if r.status().is_success() => { - refresh_vm(&db, &client, host, &vm.id).await; - } - Ok(r) => { - eprintln!( - "[ctl] WARN: agent {} returned {} when stopping VM {}", - host.addr, - r.status(), - vm.id - ); - } - Err(e) => { - eprintln!( - "[ctl] WARN: failed to contact agent {} to stop VM {}: {e}", - host.addr, vm.id - ); - } - } - } - } - - env.state = EnvState::Stopped; - let db = db.lock_db(); - let _ = db.put_env(&env); - - (StatusCode::OK, Json(ApiRespEmpty::ok())) -} - -/// POST /api/envs/:id/start -pub async fn start_env(State(db): State, Path(id): Path) -> impl IntoResponse { - let (mut env, vms, hosts) = { - let db = db.lock_db(); - let env = match db.get_env(&id) { - Ok(Some(e)) => e, - _ => { - return ( - StatusCode::NOT_FOUND, - Json(ApiRespEmpty::err(format!("environment not found: {id}"))), - ); - } - }; - let vms = db.vms_by_env(&id).unwrap_or_default(); - let hosts = db.list_hosts().unwrap_or_default(); - (env, vms, hosts) - }; - - let client = agent_client(db.api_key.as_deref(), 30); - for vm in &vms { - if let Some(host) = hosts.iter().find(|h| h.id == vm.host_id) { - let url = format!("http://{}/api/vms/{}/start", host.addr, vm.id); - match client.post(&url).send().await { - Ok(r) if r.status().is_success() => { - refresh_vm(&db, &client, host, &vm.id).await; - } - Ok(r) => { - eprintln!( - "[ctl] WARN: agent {} returned {} when starting VM {}", - host.addr, - r.status(), - vm.id - ); - } - Err(e) => { - eprintln!( - "[ctl] WARN: failed to contact agent {} to start VM {}: {e}", - host.addr, vm.id - ); - } - } - } - } - - env.state = EnvState::Active; - let db = db.lock_db(); - let _ = db.put_env(&env); - - (StatusCode::OK, Json(ApiRespEmpty::ok())) -} - -// ── Images ────────────────────────────────────────────────────────── - -/// GET /api/images -pub async fn list_images(State(db): State) -> impl IntoResponse { - let hosts = { - let db = db.lock_db(); - db.list_hosts().unwrap_or_default() - }; - - let client = agent_client(db.api_key.as_deref(), 30); - let mut images = Vec::new(); - - for host in &hosts { - if host.state != HostState::Online { - continue; - } - let url = format!("http://{}/api/images", host.addr); - if let Ok(resp) = client.get(&url).send().await - && let Ok(body) = resp.json::>>().await - && let Some(names) = body.data - { - for name in names { - images.push(ImageInfo { - name, - host_id: host.id.clone(), - }); - } - } - } - - Json(ApiResp::success(images)) -} - -// ── Status ────────────────────────────────────────────────────────── - -/// GET /api/status -pub async fn fleet_status(State(db): State) -> impl IntoResponse { - let client = agent_client(db.api_key.as_deref(), 30); - refresh_all_hosts(&db, &client).await; - - let db = db.lock_db(); - match db.fleet_status() { - Ok(s) => Json(ApiResp::success(s)), - Err(e) => Json(ApiResp::::err(e.to_string())), - } -} - -// ── VM Lookup ─────────────────────────────────────────────────────── - -/// GET /api/vms/:id — get a single VM by ID (across all hosts). -pub async fn get_vm(State(db): State, Path(id): Path) -> impl IntoResponse { - let db = db.lock_db(); - match db.get_vm(&id) { - Ok(Some(vm)) => (StatusCode::OK, Json(ApiResp::success(vm))), - Ok(None) => ( - StatusCode::NOT_FOUND, - Json(ApiResp::::err(format!("VM not found: {id}"))), - ), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(ApiResp::::err(e.to_string())), - ), - } -} - -// ── Helpers ───────────────────────────────────────────────────────── - -fn now() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} - -/// Fetch available images from all online hosts. -async fn fetch_host_images( - hosts: &[Host], - client: &reqwest::Client, -) -> HashMap> { - let mut result = HashMap::new(); - for host in hosts { - if host.state != HostState::Online { - continue; - } - let url = format!("http://{}/api/images", host.addr); - if let Ok(resp) = client.get(&url).send().await - && let Ok(body) = resp.json::>>().await - && let Some(names) = body.data - { - result.insert(host.id.clone(), names.into_iter().collect()); - } - } - result -} - -/// Refresh a single VM's state from the agent and update the controller DB. -async fn refresh_vm(state: &CtlState, client: &reqwest::Client, host: &Host, vm_id: &str) { - let url = format!("http://{}/api/vms/{}", host.addr, vm_id); - if let Ok(resp) = client.get(&url).send().await - && let Ok(body) = resp.json::>().await - && let Some(vm) = body.data - { - let db = state.lock_db(); - let _ = db.put_vm(&vm); - } -} - -/// Refresh resource snapshots for all hosts from their agents. -pub async fn refresh_all_hosts(state: &CtlState, client: &reqwest::Client) { - let hosts = { - let db = state.lock_db(); - db.list_hosts().unwrap_or_default() - }; - - for host in &hosts { - let url = format!("http://{}/api/info", host.addr); - let mut updated = host.clone(); - - match client.get(&url).send().await { - Ok(resp) => { - if let Ok(body) = resp.json::>().await - && let Some(info) = body.data - { - updated.resource = info.resource; - updated.state = HostState::Online; - } - } - Err(_) => { - updated.state = HostState::Offline; - } - } - - let db = state.lock_db(); - let _ = db.put_host(&updated); - } -} diff --git a/crates/ctl/src/main.rs b/crates/ctl/src/main.rs deleted file mode 100644 index 112376b..0000000 --- a/crates/ctl/src/main.rs +++ /dev/null @@ -1,203 +0,0 @@ -//! TTstack central controller entry point. -//! -//! The controller manages the fleet of hosts, schedules VM placement, -//! and exposes an HTTP API for the CLI client and web interface. - -mod auth; -mod config; -mod db; -mod handler; -mod scheduler; -mod web; - -use axum::Router; -use axum::routing::{get, post}; -use clap::Parser; -use config::Config; -use db::Db; -use handler::CtlState; -use std::sync::Arc; - -#[tokio::main] -async fn main() { - let cfg = Config::parse(); - - std::fs::create_dir_all(&cfg.data_dir).unwrap_or_else(|e| { - eprintln!("Failed to create data dir {}: {e}", cfg.data_dir); - std::process::exit(1); - }); - - let db_path = format!("{}/ctl.db", cfg.data_dir); - let db = Db::open(&db_path).unwrap_or_else(|e| { - eprintln!("Failed to open database: {e}"); - std::process::exit(1); - }); - - let state: CtlState = Arc::new(handler::CtlShared::new(db, cfg.api_key.clone())); - - // Background task: expire old environments - let expiry_state = state.clone(); - tokio::spawn(async move { - loop { - tokio::time::sleep(std::time::Duration::from_secs(60)).await; - expire_envs(&expiry_state).await; - } - }); - - // Background task: periodic host health check - let heartbeat_state = state.clone(); - tokio::spawn(async move { - let client = handler::agent_client(heartbeat_state.api_key.as_deref(), 10); - loop { - tokio::time::sleep(std::time::Duration::from_secs(30)).await; - handler::refresh_all_hosts(&heartbeat_state, &client).await; - } - }); - - if cfg.api_key.is_some() { - eprintln!("API key authentication enabled"); - } else { - eprintln!("WARNING: no --api-key set, all API endpoints are unauthenticated!"); - } - - let api_routes = Router::new() - .route( - "/api/hosts", - get(handler::list_hosts).post(handler::register_host), - ) - .route( - "/api/hosts/{id}", - get(handler::get_host).delete(handler::remove_host), - ) - .route( - "/api/envs", - get(handler::list_envs).post(handler::create_env), - ) - .route( - "/api/envs/{id}", - get(handler::get_env).delete(handler::delete_env), - ) - .route("/api/envs/{id}/stop", post(handler::stop_env)) - .route("/api/envs/{id}/start", post(handler::start_env)) - .route("/api/vms/{id}", get(handler::get_vm)) - .route("/api/images", get(handler::list_images)) - .route("/api/status", get(handler::fleet_status)) - .with_state(state); - - let api_routes = if let Some(ref key) = cfg.api_key { - api_routes.layer(axum::middleware::from_fn(auth::make_auth_layer( - key.clone(), - ))) - } else { - api_routes - }; - - let app = Router::new().route("/", get(web::index)).merge(api_routes); - - let listener = tokio::net::TcpListener::bind(&cfg.listen) - .await - .unwrap_or_else(|e| { - eprintln!("Failed to bind {}: {e}", cfg.listen); - std::process::exit(1); - }); - - eprintln!("tt-ctl listening on {}", cfg.listen); - - axum::serve(listener, app) - .with_graceful_shutdown(shutdown_signal()) - .await - .unwrap_or_else(|e| eprintln!("Server error: {e}")); - - eprintln!("tt-ctl shutting down"); -} - -async fn shutdown_signal() { - let _ = tokio::signal::ctrl_c().await; - eprintln!("received shutdown signal"); -} - -/// Maximum retries when contacting an agent during expiry cleanup. -const EXPIRY_RETRIES: u32 = 2; - -/// Periodically destroy expired environments. -async fn expire_envs(state: &CtlState) { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - - let expired = { - let db = state.lock_db(); - db.list_envs() - .unwrap_or_default() - .into_iter() - .filter(|e| e.expires_at > 0 && e.expires_at <= now) - .map(|e| e.id) - .collect::>() - }; - - let client = handler::agent_client(state.api_key.as_deref(), 15); - - for env_id in expired { - eprintln!("expiring environment: {env_id}"); - - let (vms, hosts) = { - let db = state.lock_db(); - let vms = db.vms_by_env(&env_id).unwrap_or_default(); - let hosts = db.list_hosts().unwrap_or_default(); - (vms, hosts) - }; - - for vm in &vms { - if let Some(host) = hosts.iter().find(|h| h.id == vm.host_id) { - let url = format!("http://{}/api/vms/{}", host.addr, vm.id); - let mut ok = false; - for attempt in 0..=EXPIRY_RETRIES { - match client.delete(&url).send().await { - Ok(r) if r.status().is_success() => { - ok = true; - break; - } - Ok(r) => { - eprintln!( - "[ctl] WARN: agent {} returned {} deleting VM {} (attempt {}/{})", - host.addr, - r.status(), - vm.id, - attempt + 1, - EXPIRY_RETRIES + 1 - ); - } - Err(e) => { - eprintln!( - "[ctl] WARN: failed to reach {} to delete VM {} (attempt {}/{}): {e}", - host.addr, - vm.id, - attempt + 1, - EXPIRY_RETRIES + 1 - ); - } - } - if attempt < EXPIRY_RETRIES { - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - } - } - if !ok { - eprintln!( - "[ctl] ERROR: could not delete VM {} on host {} after {} attempts; \ - VM may be orphaned", - vm.id, - host.addr, - EXPIRY_RETRIES + 1 - ); - } - } - } - - let db = state.lock_db(); - for vm in &vms { - let _ = db.remove_vm(&vm.id); - } - let _ = db.remove_env(&env_id); - } -} diff --git a/crates/ctl/src/scheduler.rs b/crates/ctl/src/scheduler.rs deleted file mode 100644 index 001707a..0000000 --- a/crates/ctl/src/scheduler.rs +++ /dev/null @@ -1,309 +0,0 @@ -//! VM placement scheduler. -//! -//! Decides which host should run each VM based on available resources, -//! supported engines, image availability, and a simple best-fit strategy. - -use ruc::*; -use std::collections::{HashMap, HashSet}; -use ttcore::api::VmSpec; -use ttcore::model::*; - -/// Result of scheduling: VM spec + chosen host. -#[derive(Debug)] -pub struct Placement { - pub host_id: String, - pub host_addr: String, -} - -/// Choose the best host for a VM spec using a best-fit strategy. -/// -/// Prefers the host with the least free resources that can still -/// accommodate the VM, to pack hosts densely and leave larger hosts -/// available for bigger workloads. -/// -/// `host_images` maps host_id → set of available image names. -/// If the map is empty, image validation is skipped (for backward compat). -pub fn place_vm( - hosts: &[Host], - spec: &VmSpec, - host_images: &HashMap>, -) -> Result { - let cpu = spec.cpu.unwrap_or(VM_CPU_DEFAULT); - let mem = spec.mem.unwrap_or(VM_MEM_DEFAULT); - let disk = spec.disk.unwrap_or(VM_DISK_DEFAULT); - - // Docker images are managed by Docker, not by the image directory - let check_images = !host_images.is_empty() && spec.engine != Engine::Docker; - - let mut candidates: Vec<&Host> = hosts - .iter() - .filter(|h| { - h.state == HostState::Online - && h.engines.contains(&spec.engine) - && h.resource.can_fit(cpu, mem, disk) - && (!check_images - || host_images - .get(&h.id) - .is_some_and(|imgs| imgs.contains(&spec.image))) - }) - .collect(); - - if candidates.is_empty() { - // Provide a more helpful error message - let online = hosts - .iter() - .filter(|h| h.state == HostState::Online) - .count(); - let with_engine = hosts - .iter() - .filter(|h| h.state == HostState::Online && h.engines.contains(&spec.engine)) - .count(); - let with_resource = hosts - .iter() - .filter(|h| { - h.state == HostState::Online - && h.engines.contains(&spec.engine) - && h.resource.can_fit(cpu, mem, disk) - }) - .count(); - - if online == 0 { - return Err(eg!("no online hosts available")); - } else if with_engine == 0 { - return Err(eg!("no online host supports engine={}", spec.engine,)); - } else if with_resource == 0 { - return Err(eg!( - "no host has enough resources for engine={}, cpu={}, mem={}MB, disk={}MB", - spec.engine, - cpu, - mem, - disk, - )); - } else { - return Err(eg!( - "no host has image '{}' for engine={}", - spec.image, - spec.engine, - )); - } - } - - // Sort by free memory ascending (best-fit) - candidates.sort_by_key(|h| h.resource.mem_free()); - - let host = candidates[0]; - Ok(Placement { - host_id: host.id.clone(), - host_addr: host.addr.clone(), - }) -} - -/// Schedule an entire environment's VMs across the fleet. -/// -/// Returns a list of (VmSpec, Placement) pairs. -pub fn schedule_env( - hosts: &[Host], - specs: &[VmSpec], - host_images: &HashMap>, -) -> Result> { - let mut result = Vec::with_capacity(specs.len()); - - // Work with a mutable copy of host resources for multi-VM scheduling - let mut shadow: Vec = hosts.to_vec(); - - for spec in specs { - let placement = place_vm(&shadow, spec, host_images)?; - - // Update shadow resources to account for this allocation - if let Some(h) = shadow.iter_mut().find(|h| h.id == placement.host_id) { - let cpu = spec.cpu.unwrap_or(VM_CPU_DEFAULT); - let mem = spec.mem.unwrap_or(VM_MEM_DEFAULT); - let disk = spec.disk.unwrap_or(VM_DISK_DEFAULT); - h.resource.cpu_used += cpu; - h.resource.mem_used += mem; - h.resource.disk_used += disk; - h.resource.vm_count += 1; - } - - result.push((spec.clone(), placement)); - } - - Ok(result) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn make_host(id: &str, cpu: u32, mem: u32, engines: Vec) -> Host { - Host { - id: id.into(), - addr: format!("{id}:9100"), - resource: Resource { - cpu_total: cpu, - cpu_used: 0, - mem_total: mem, - mem_used: 0, - disk_total: 500_000, - disk_used: 0, - vm_count: 0, - }, - state: HostState::Online, - engines, - storage: Storage::File, - registered_at: 0, - } - } - - fn make_spec() -> VmSpec { - VmSpec { - image: "ubuntu".into(), - engine: Engine::Qemu, - cpu: Some(2), - mem: Some(1024), - disk: Some(40960), - ports: vec![22], - deny_outgoing: false, - ssh_keys: vec![], - } - } - - fn empty_images() -> HashMap> { - HashMap::new() - } - - fn images_for(host_id: &str, imgs: &[&str]) -> HashMap> { - let mut m = HashMap::new(); - m.insert(host_id.into(), imgs.iter().map(|s| s.to_string()).collect()); - m - } - - #[test] - fn place_vm_picks_online_host() { - let hosts = vec![make_host("h1", 8, 16384, vec![Engine::Qemu])]; - let p = place_vm(&hosts, &make_spec(), &empty_images()).unwrap(); - assert_eq!(p.host_id, "h1"); - } - - #[test] - fn place_vm_skips_offline_host() { - let mut h = make_host("h1", 8, 16384, vec![Engine::Qemu]); - h.state = HostState::Offline; - let hosts = vec![h]; - assert!(place_vm(&hosts, &make_spec(), &empty_images()).is_err()); - } - - #[test] - fn place_vm_skips_wrong_engine() { - let hosts = vec![make_host("h1", 8, 16384, vec![Engine::Docker])]; - let spec = make_spec(); // wants Qemu - assert!(place_vm(&hosts, &spec, &empty_images()).is_err()); - } - - #[test] - fn place_vm_skips_insufficient_resources() { - let hosts = vec![make_host("h1", 1, 512, vec![Engine::Qemu])]; - let spec = make_spec(); // needs 2 CPU, 1024 mem - assert!(place_vm(&hosts, &spec, &empty_images()).is_err()); - } - - #[test] - fn place_vm_best_fit_prefers_smaller() { - // h1 has more room, h2 has less but enough - let hosts = vec![ - make_host("h1", 16, 32768, vec![Engine::Qemu]), - make_host("h2", 4, 4096, vec![Engine::Qemu]), - ]; - let p = place_vm(&hosts, &make_spec(), &empty_images()).unwrap(); - assert_eq!(p.host_id, "h2"); // best-fit picks smaller - } - - #[test] - fn place_vm_checks_image_availability() { - let hosts = vec![make_host("h1", 8, 16384, vec![Engine::Qemu])]; - let imgs = images_for("h1", &["alpine"]); - let spec = make_spec(); // wants "ubuntu" - let result = place_vm(&hosts, &spec, &imgs); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("image")); - } - - #[test] - fn place_vm_with_matching_image() { - let hosts = vec![make_host("h1", 8, 16384, vec![Engine::Qemu])]; - let imgs = images_for("h1", &["ubuntu", "alpine"]); - let p = place_vm(&hosts, &make_spec(), &imgs).unwrap(); - assert_eq!(p.host_id, "h1"); - } - - #[test] - fn place_vm_skips_image_check_for_docker() { - let hosts = vec![make_host("h1", 8, 16384, vec![Engine::Docker])]; - let imgs = images_for("h1", &["alpine"]); // no "ubuntu" - let mut spec = make_spec(); - spec.engine = Engine::Docker; - // Should succeed — Docker images are not checked against host_images - let p = place_vm(&hosts, &spec, &imgs).unwrap(); - assert_eq!(p.host_id, "h1"); - } - - #[test] - fn schedule_env_distributes_when_full() { - let hosts = vec![ - make_host("h1", 4, 4096, vec![Engine::Qemu]), - make_host("h2", 4, 4096, vec![Engine::Qemu]), - ]; - - // 3 VMs each needing 2 CPU: h1 takes 2 (filling up), h2 takes 1 - let specs: Vec = (0..3).map(|_| make_spec()).collect(); - let placements = schedule_env(&hosts, &specs, &empty_images()).unwrap(); - assert_eq!(placements.len(), 3); - - let on_h1 = placements.iter().filter(|(_, p)| p.host_id == "h1").count(); - let on_h2 = placements.iter().filter(|(_, p)| p.host_id == "h2").count(); - assert_eq!(on_h1, 2); - assert_eq!(on_h2, 1); - } - - #[test] - fn schedule_env_fails_if_no_capacity() { - let hosts = vec![make_host("h1", 2, 2048, vec![Engine::Qemu])]; - let specs: Vec = (0..2).map(|_| make_spec()).collect(); - assert!(schedule_env(&hosts, &specs, &empty_images()).is_err()); - } - - #[test] - fn schedule_env_empty_specs_ok() { - let hosts = vec![make_host("h1", 8, 16384, vec![Engine::Qemu])]; - let placements = schedule_env(&hosts, &[], &empty_images()).unwrap(); - assert!(placements.is_empty()); - } - - #[test] - fn error_message_no_online() { - let mut h = make_host("h1", 8, 16384, vec![Engine::Qemu]); - h.state = HostState::Offline; - let err = place_vm(&[h], &make_spec(), &empty_images()) - .unwrap_err() - .to_string(); - assert!(err.contains("no online hosts")); - } - - #[test] - fn error_message_no_engine() { - let hosts = vec![make_host("h1", 8, 16384, vec![Engine::Docker])]; - let err = place_vm(&hosts, &make_spec(), &empty_images()) - .unwrap_err() - .to_string(); - assert!(err.contains("engine=qemu")); - } - - #[test] - fn error_message_no_resource() { - let hosts = vec![make_host("h1", 1, 512, vec![Engine::Qemu])]; - let err = place_vm(&hosts, &make_spec(), &empty_images()) - .unwrap_err() - .to_string(); - assert!(err.contains("resources")); - } -} diff --git a/crates/ctl/src/web.rs b/crates/ctl/src/web.rs deleted file mode 100644 index 1b0bf26..0000000 --- a/crates/ctl/src/web.rs +++ /dev/null @@ -1,467 +0,0 @@ -//! Embedded web frontend for TTstack management. -//! -//! Serves a single-page application directly from the controller binary. -//! No external files or build tools required. - -use axum::response::Html; - -/// GET / — serve the management dashboard. -/// -/// The page itself requires no authentication. When API key auth is -/// enabled, the JS client detects 401 responses and prompts the user -/// to enter the key (stored in sessionStorage). -pub async fn index() -> Html<&'static str> { - Html(FRONTEND_HTML) -} - -const FRONTEND_HTML: &str = r##" - - - - -TTstack — Dashboard - - - -
-
-

TTstack Dashboard

- -
- - -
-
Loading...
-
- - - - - - - - - -
- - - - - - - -
- - - - -"##; diff --git a/docs/compatibility.md b/docs/compatibility.md deleted file mode 100644 index c90aaf6..0000000 --- a/docs/compatibility.md +++ /dev/null @@ -1,261 +0,0 @@ -# TTstack Compatibility Matrix - -This document describes the host OS and guest VM/container compatibility -for TTstack, including platforms that have been tested in practice and -those planned for support. - ---- - -## Host OS Compatibility - -TTstack agents and the controller can run on the following host -operating systems. The table below distinguishes between platforms -that have been **tested** (with automated or manual verification) and -those that are **planned** (expected to work based on architecture but -not yet verified in CI or lab environments). - -| Host OS | Version | Status | Storage Backends | Engines | Notes | -|---------|---------|--------|-----------------|---------|-------| -| Debian | 13 (Trixie) | **Tested** | file, zvol | QEMU/KVM, Firecracker, Docker | Primary development platform | -| Alpine Linux | 3.23 | **Tested** | file | QEMU/KVM, Docker | Requires `iproute2`, `nftables`, `socat`, `qemu-system-x86_64`; BusyBox `ip` is not sufficient | -| Ubuntu | 24.04 LTS | Planned | file, zvol | QEMU/KVM, Firecracker, Docker | Debian-derivative; expected full compatibility | -| Rocky Linux | 10.x | Planned | file, zvol | QEMU/KVM, Firecracker, Docker/Podman | RHEL-derivative; nftables is the default firewall backend | -| Gentoo | 23.0 | Planned | file, zvol | QEMU/KVM, Firecracker, Docker | Requires manual package installation | -| FreeBSD | 14.3 | **Tested** | file, zvol | Bhyve, Jail | Uses PF instead of nftables; clang from base system | - -### Host OS Requirements - -All Linux hosts require: - -- **Kernel**: 5.10+ (for KVM, nftables, and namespace support) -- **nftables**: For NAT and port forwarding (replaces legacy iptables) -- **iproute2**: Full `ip` command (BusyBox `ip` lacks `tuntap` support) -- **socat**: For QEMU monitor socket communication -- **bridge-utils** or iproute2 bridge support -- **SQLite 3**: Linked at build time via `rusqlite` (bundled by default) - -Engine-specific requirements: - -| Engine | Required Packages | Kernel Features | -|--------|------------------|-----------------| -| QEMU/KVM | `qemu-system-x86_64`, `socat` | `/dev/kvm` (hardware virtualization) | -| Firecracker | `firecracker` binary | `/dev/kvm`, `tun` module | -| Docker | `docker` or `podman` | cgroups v2, overlayfs | - -FreeBSD hosts require: - -- **PF**: Packet filter for NAT (configured automatically) -- **Bhyve**: Native hypervisor (FreeBSD 10+) -- **Jail**: Native container isolation - -### Storage Backend Requirements - -| Backend | Host Requirements | Performance | Notes | -|---------|------------------|-------------|-------| -| file | Any filesystem | Baseline | Linux: `cp --reflink=auto` for CoW; FreeBSD: `cp -a` | -| zvol | ZFS pool mounted | Fast (instant clone) | Requires `zfs` CLI; images stored as zvols (raw block devices at `/dev/zvol/...`) | - -### Tested Host Configurations - -The following configurations have been verified in practice: - -#### Debian 13 (Trixie) — x86_64 - -- **Kernel**: 6.12.73+deb13 -- **CPU**: 64 cores, **RAM**: 125 GiB -- **QEMU**: 10.0.7 with KVM acceleration -- **Firecracker**: Tested with fc-alpine image -- **Docker**: 26.1.5 -- **Storage**: file (default), zvol (pool `ttpool`) -- **Networking**: nftables with tt-nat table, bridge `tt0` - -**Test results**: -- All storage backends (file, zvol): VM create/destroy with proper clone/snapshot lifecycle -- QEMU engine: Full lifecycle (create → run → pause → resume → destroy) -- Firecracker engine: Full lifecycle (create → run → pause → destroy) -- Docker engine: Full lifecycle (create → run → stop → start → destroy) with native port mapping -- Concurrent environment creation (5 parallel): No conflicts -- Agent crash recovery: VM state and host_id persist across forced restarts -- Multi-host scheduling: VMs distributed across hosts when single host is insufficient - -#### Alpine Linux 3.23 — x86_64 - -- **Kernel**: 6.18.9-0-lts -- **CPU**: 128 cores, **RAM**: 252 GiB -- **QEMU**: 10.1.3 with KVM acceleration -- **Docker**: 29.1.3 -- **Storage**: file -- **Networking**: nftables + iproute2 (required; BusyBox `ip` is insufficient) - -**Required packages** (beyond base install): -``` -apk add qemu-system-x86_64 qemu-img docker socat nftables iproute2 curl -modprobe tun # if /dev/net/tun is missing -``` - -**Test results**: -- QEMU engine: Full lifecycle (create → run → pause → resume → destroy) -- Multi-host: Successfully participates as remote agent in distributed fleet -- Cross-host scheduling: VMs correctly placed and managed from central controller - -#### FreeBSD 14.3-RELEASE — x86_64 - -- **Kernel**: 14.3-RELEASE GENERIC -- **Clang**: 19.1.7 (from base system) -- **Rust**: 1.92.0 (from pkg) -- **Storage**: file -- **Engines detected**: bhyve, jail - -**Test results**: -- All 82 unit tests pass natively on FreeBSD (89 workspace-wide; some Linux-only tests excluded) -- Agent: Starts successfully, bridge `tt0` created via `ifconfig bridge create name tt0` -- Jail engine: Full lifecycle (create → stop → start → destroy) verified end-to-end -- Bhyve engine: Code verified; requires hardware VMX (cannot test in nested QEMU) -- CLI: Connects to remote controller, displays fleet status and host list -- Controller: Builds and runs on FreeBSD (cross-platform component) -- Build: Compiles all three binaries (tt, tt-agent, tt-ctl) without modification - ---- - -## Guest VM / Container Compatibility - -### Container Engines (Docker / Podman) - -TTstack delegates container management entirely to Docker or Podman. -Any container image that runs on the host's container runtime is -supported. This includes the full OCI container ecosystem: - -| Category | Examples | Notes | -|----------|----------|-------| -| Official base images | `alpine`, `debian`, `ubuntu`, `rockylinux`, `fedora` | All tags supported | -| Language runtimes | `python`, `node`, `golang`, `rust`, `ruby`, `openjdk` | | -| Databases | `postgres`, `mysql`, `redis`, `mongodb`, `mariadb` | | -| Web servers | `nginx`, `httpd`, `caddy`, `traefik` | | -| Application platforms | `wordpress`, `grafana`, `prometheus`, `gitlab` | | -| Custom images | Any `Dockerfile`-built or registry image | | - -**Tested container images**: -- `alpine:3.21` (minimal 754 KB test image) -- Custom `tt-test-alpine` (init-based test container) - -### VM Engines (QEMU/KVM, Firecracker, Bhyve) - -VM engines boot full operating system images. The guest OS must be -compatible with the virtual hardware presented by each engine. - -#### QEMU/KVM Guest Compatibility - -QEMU provides full x86_64 hardware emulation with KVM acceleration. -Any operating system that supports the `virtio` device family is -recommended for best performance. - -**Supported guest OS families** (server editions, releases within the -last 3 years): - -| Guest OS | Versions | Disk Format | Notes | -|----------|----------|-------------|-------| -| Debian | 11 (Bullseye), 12 (Bookworm), 13 (Trixie) | qcow2 | Excellent virtio support | -| Ubuntu Server | 22.04 LTS, 24.04 LTS | qcow2 | Cloud images work directly | -| Alpine Linux | 3.18–3.23 | qcow2 | Minimal footprint, fast boot | -| Rocky Linux | 8.x, 9.x, 10.x | qcow2 | RHEL-compatible | -| Alma Linux | 8.x, 9.x | qcow2 | RHEL-compatible | -| Fedora Server | 39, 40, 41 | qcow2 | Cutting-edge kernel | -| openSUSE Leap | 15.5, 15.6 | qcow2 | Enterprise-grade | -| Arch Linux | Rolling | qcow2 | Latest packages | -| Gentoo | 23.0 | qcow2 | Source-based | -| FreeBSD | 13.x, 14.x | qcow2 | Full virtio support since 12.0 | -| Windows Server | 2019, 2022, 2025 | qcow2 | Requires virtio-win drivers | - -**Tested guest images**: -- Minimal qcow2 test image (256 MiB, lifecycle verification) -- Firecracker Alpine rootfs (kernel + ext4 rootfs) - -**Guest image requirements**: -- Format: qcow2 (recommended) or raw -- For directory-based storage (zvol): images are ZFS zvols exposed as raw block devices -- The engine automatically resolves `disk.qcow2` inside directories - -#### Firecracker Guest Compatibility - -Firecracker boots microVMs with a Linux kernel and root filesystem. -It does **not** support full BIOS/UEFI boot — the guest must be a -Linux kernel (`vmlinux`) paired with an ext4 root filesystem. - -| Guest OS | Versions | Image Format | Notes | -|----------|----------|-------------|-------| -| Alpine Linux | 3.18–3.23 | vmlinux + rootfs.ext4 | Ideal for microVMs | -| Debian | 11–13 | vmlinux + rootfs.ext4 | Requires kernel extraction | -| Ubuntu | 22.04, 24.04 | vmlinux + rootfs.ext4 | Cloud kernel works | -| Amazon Linux | 2, 2023 | vmlinux + rootfs.ext4 | Native Firecracker support | - -**Firecracker image structure**: -``` -images/fc-alpine/ -├── vmlinux # Uncompressed Linux kernel -└── rootfs.ext4 # Root filesystem -``` - -**Tested**: Alpine Linux microVM image with custom init. - -#### Bhyve Guest Compatibility (FreeBSD hosts only) - -Bhyve is FreeBSD's native hypervisor. It supports: - -| Guest OS | Versions | Notes | -|----------|----------|-------| -| FreeBSD | 12.x–14.x | Native guest support | -| Linux | Recent kernels (5.x+) | Requires UEFI boot or grub-bhyve | -| OpenBSD | 7.x | | -| Windows | 10, 11, Server 2019+ | Experimental | - -**Tested**: Build and unit tests pass on FreeBSD 14.3. Agent starts -and reports bhyve/jail engines. CLI and controller work correctly. - ---- - -## Network Compatibility - -| Feature | Linux (nftables) | FreeBSD (PF) | -|---------|------------------|--------------| -| Bridge networking | `tt0` bridge (10.10.0.1/16) | `tt0` bridge | -| TAP devices | Per-VM, auto-named | Per-VM | -| NAT / masquerade | nftables `tt-nat` table | PF rules | -| Port forwarding (DNAT) | nftables prerouting chain | PF rdr rules | -| Outgoing traffic block | nftables forward chain + denylist set | PF block rules | -| Docker networking | Native Docker `-p` (no nftables) | Native Docker | - -**Tested**: Full nftables networking on Debian 13 and Alpine 3.23, -including port forwarding, NAT, deny-outgoing, and Docker-native -port publishing. Bridge and TAP creation on FreeBSD 14.3. - ---- - -## Build Toolchain - -| Component | Minimum Version | Tested Version | -|-----------|----------------|----------------| -| Rust | 1.86 (edition 2024) | 1.92.0–1.93.1 | -| Cargo | Matching Rust | 1.92.0–1.93.1 | -| C compiler | GCC or Clang | GCC 14.2 (Debian), GCC 15.2 (Alpine), Clang 19.1 (FreeBSD) | -| SQLite | 3.x (bundled) | Bundled via `libsqlite3-sys` | -| OpenSSL | 1.1+ or 3.x | 3.5.x | - -**Alpine build note**: Requires `openssl-dev` and `openssl-libs-static` -for static linking. For musl cross-compilation from Debian: -`OPENSSL_STATIC=1 OPENSSL_NO_VENDOR=0 cargo build --release --target x86_64-unknown-linux-musl --features reqwest/native-tls-vendored` - -**FreeBSD build note**: Requires `rust`, `pkgconf`, and `openssl` from pkg. -FreeBSD 14.3 ships clang 19 in base — no additional compiler setup needed. - ---- - -## Summary - -| Component | Tested | Planned | -|-----------|--------|---------| -| Host OS | Debian 13, Alpine 3.23, FreeBSD 14.3 | Ubuntu 24.04, Rocky 10, Gentoo 23 | -| VM engines | QEMU/KVM, Firecracker, Docker (Linux); Bhyve, Jail (FreeBSD) | — | -| Storage backends | file, zvol | All verified on Debian 13 | -| Multi-host | 3-host fleet (Debian + Alpine + FreeBSD) | Up to 50 hosts | -| Guest OS (containers) | Alpine-based test images | Full OCI ecosystem | -| Guest OS (VMs) | Minimal test images | See guest compatibility tables above | diff --git a/docs/deployment.md b/docs/deployment.md deleted file mode 100644 index be2d9e0..0000000 --- a/docs/deployment.md +++ /dev/null @@ -1,142 +0,0 @@ -# Deployment Guide - -Deployment is built into the `tt` CLI binary — no external scripts needed. - -## Prerequisites - -**Linux agents**: -- nftables -- Kernel modules: `tun`, `vhost_net`, `kvm_intel` (or `kvm_amd`) -- `socat` (for QEMU monitor communication) -- `genisoimage` or `mkisofs` (for cloud-init seed ISO generation) - -**FreeBSD agents**: -- PF enabled - -## Local Deploy - -```bash -# Deploy both agent and controller (auto-generates API key): -sudo tt deploy all - -# Or deploy separately: -sudo tt deploy agent -sudo tt deploy ctl - -# Specify a custom binary directory: -sudo tt deploy all --release-dir ./target/release -``` - -The API key is printed on completion. Save it for CLI configuration: - -```bash -tt config 127.0.0.1:9200 --api-key -``` - -## Distributed Deploy - -For multi-host fleets, create a `deploy.toml` config: - -```bash -cp tools/deploy.toml.example deploy.toml -# Edit with your fleet IPs... -tt deploy dist deploy.toml -``` - -### deploy.toml reference - -```toml -[general] -prefix = "/opt/ttstack" # Install path on all hosts -user = "ttstack" # Runtime user (created if absent) -release_dir = "./target/release" # Local path to compiled binaries -# api_key = "my-secret-key" # Optional; auto-generated if omitted - -[controller] -host = "10.0.0.1" # Controller IP or hostname -# ssh_user = "root" # SSH user (default: root) -# ssh_port = 22 # SSH port (default: 22) -# listen = "0.0.0.0:9200" # Listen address (default: 0.0.0.0:9200) -# data_dir = "/home/ttstack/ctl" # Database directory - -[[agents]] -host = "10.0.0.2" # Agent IP or hostname -# ssh_user = "root" -# ssh_port = 22 -# listen = "0.0.0.0:9100" -# storage = "file" # "file" or "zvol" -# image_dir = "/home/ttstack/images" -# runtime_dir = "/home/ttstack/runtime" -# cpu_total = 0 # 0 = auto-detect -# mem_total = 0 # MiB, 0 = auto-detect -# disk_total = "200G" # MiB or "NNNG" shorthand -# host_id = "my-node" # Custom host ID -# release_dir = "./target/release-musl" # Per-agent binary path override - -[[agents]] -host = "10.0.0.3" -storage = "zvol" -image_dir = "tank/ttstack/images" -runtime_dir = "tank/ttstack/runtime" -cpu_total = 32 -mem_total = 65536 # 64 GiB -disk_total = "1000G" # ~1 TiB -``` - -### Cross-platform notes - -- **Alpine Linux**: Use `release_dir` per-agent to point to musl-compiled binaries -- **Systemd** (Debian/Ubuntu/Rocky): Auto-detected; creates systemd units -- **OpenRC** (Alpine): Auto-detected; creates `/etc/init.d/` scripts -- **FreeBSD**: Falls back to manual start if no init system detected - -## Directory Layout - -``` -/opt/ttstack/bin/ # binaries (tt, tt-ctl, tt-agent) -/home/ttstack/ # runtime data (dedicated ttstack user) - ├── images/ # base VM/container images - ├── runtime/ # transient VM image clones - ├── data/ # agent SQLite database - ├── ctl/ # controller SQLite database - └── run/ # PID files, sockets, seed ISOs -``` - -## Idempotent Upgrades - -Deploy is idempotent — re-running copies new binaries, restarts services, -and preserves all data. Schema migrations run automatically on startup. - -```bash -# Rebuild and re-deploy: -make release -tt deploy dist deploy.toml -``` - -## Agent Configuration - -All resources in **MiB**. Set to 0 for auto-detection. - -``` -tt-agent [OPTIONS] - - --listen Listen address [0.0.0.0:9100] - --image-dir Base image directory [/home/ttstack/images] - --runtime-dir Runtime clone directory [/home/ttstack/runtime] - --data-dir Database directory [/home/ttstack/data] - --storage zvol | file [file] - --cpu-total CPU cores (0=auto) [0] - --mem-total Memory in MiB (0=auto) [0] - --disk-total Disk in MiB [204800 (~200 GiB)] - --host-id Host ID (auto-generated) -``` - -## Controller Configuration - -``` -tt-ctl [OPTIONS] - - --listen Listen address [0.0.0.0:9200] - --data-dir Database directory [/home/ttstack/ctl] - --api-key API key for auth (env: TT_API_KEY) [none] -``` diff --git a/docs/guest-images.md b/docs/guest-images.md deleted file mode 100644 index 2c5f6b4..0000000 --- a/docs/guest-images.md +++ /dev/null @@ -1,343 +0,0 @@ -# Guest Image Guide - -TTstack supports multiple VM engines, each requiring its own image format. -This guide covers how to create, manage, and deploy guest images for each engine. - -## Quick Start - -Use the built-in image recipes: - -```bash -# List available recipes -tt image recipes - -# Create all images for this platform -sudo tt image create all --image-dir /home/ttstack/images - -# Create all Docker images -sudo tt image create all --engine docker - -# Create a specific image -sudo tt image create fc-alpine --image-dir /home/ttstack/images -sudo tt image create alpine-cloud --image-dir /home/ttstack/images -``` - -## Accessing VMs - -All VMs and containers are accessed via **SSH**. When creating an environment, -provide your SSH public key(s) — TTstack injects them into the guest via -cloud-init (QEMU) or authorized_keys. Port 22 is always auto-included. - -| Engine | Access Method | -|--------|--------------| -| **QEMU** (cloud images) | SSH via port forwarding (key injected by cloud-init) | -| **QEMU** (custom images) | SSH via port forwarding (your own key setup) | -| **Docker** | SSH into container (if sshd installed) or `docker exec` from host | -| **Firecracker** | Serial console only (no SSH by default) | -| **Bhyve** (FreeBSD) | SSH via port forwarding | - -### QEMU Cloud Images (SSH) - -For built-in cloud images (`alpine-cloud`, `debian-cloud`, `ubuntu-cloud`), -TTstack auto-generates a **cloud-init seed ISO** on each VM boot that: - -- Injects your SSH public key(s) into `~root/.ssh/authorized_keys` -- Enables SSH public-key authentication -- Configures the VM's network (static IP, gateway, DNS) - -To SSH into a QEMU VM: - -```bash -# Create environment with your SSH key -tt env create myenv --image alpine-cloud --engine qemu \ - --ssh-key ~/.ssh/id_ed25519.pub - -# Show environment to see port mappings -tt env show myenv -# Example output: -# ID IMAGE ENGINE STATE IP PORTS -# abc12345-678 alpine-cloud qemu running 10.10.0.3 20100->22 - -# SSH using the mapped port -ssh root@ -p 20100 -``` - -For custom QEMU images that do not use cloud-init, the seed ISO -is harmlessly ignored — you manage SSH credentials yourself. - -### Docker Containers - -Docker containers are managed via the Docker runtime. If the container -has an SSH daemon, connect via the mapped port. Otherwise, access -from the host machine: - -```bash -# Find the container ID -docker ps | grep - -# Exec into the container (host-only fallback) -docker exec -it sh -``` - -### Firecracker MicroVMs - -Firecracker VMs boot into a shell on the serial console but have -no SSH daemon by default. They are designed for headless workloads. -To add SSH, customize the rootfs image with an OpenSSH server. - -## Image Formats by Engine - -### Docker / Podman - -Docker images are **container images** pulled from registries or built locally. -They are **not** stored in the TTstack image directory — they live in the -Docker/Podman image store. - -**Creating a Docker image:** -```bash -# Pull from a registry -docker pull alpine:latest - -# Or build locally -cat > Dockerfile <<'EOF' -FROM scratch -COPY myapp /app -CMD ["/app"] -EOF -docker build -t my-image . -``` - -**Using with TTstack:** -```bash -tt env create myenv --image alpine --engine docker --port 80 -``` - -**Key points:** -- The `--image` name must match a locally available Docker/Podman image -- Docker images do NOT need to exist in `image_dir` -- Port mappings are handled by Docker's `-p` flag -- Docker manages its own networking (no TAP devices or bridge needed) - -### Firecracker - -Firecracker images are **directories** containing two files: -- `vmlinux` — uncompressed Linux kernel with virtio_mmio built-in -- `rootfs.ext4` — ext4 filesystem image used as the root device - -**Image directory structure:** -``` -/home/ttstack/images/ - └── fc-alpine/ - ├── vmlinux # ~21MB kernel - └── rootfs.ext4 # Root filesystem -``` - -**Creating manually:** - -1. **Get a Firecracker-compatible kernel:** - ```bash - # Option A: Download pre-built (recommended) - curl -sL https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin \ - -o /home/ttstack/images/fc-alpine/vmlinux - - # Option B: Build from source with virtio_mmio=y - # (The stock Debian kernel won't work — it has virtio_mmio as a module) - ``` - -2. **Create the rootfs:** - ```bash - dd if=/dev/zero of=rootfs.ext4 bs=1M count=128 - mkfs.ext4 rootfs.ext4 - mkdir /tmp/mnt && mount -o loop rootfs.ext4 /tmp/mnt - - # Populate with your desired userspace (Alpine, BusyBox, etc.) - # At minimum: /sbin/init (or /init) must exist - mkdir -p /tmp/mnt/{bin,sbin,etc,proc,sys,dev} - # ... install packages or copy a static init binary ... - - umount /tmp/mnt - ``` - -**Important notes:** -- The kernel MUST have `virtio_mmio` built-in (not as a module) -- The stock Debian/Ubuntu kernel will NOT work — use the Firecracker pre-built kernel -- Boot args: `console=ttyS0 reboot=k panic=1 pci=off`; the rootfs is set via `is_root_device: true` -- Firecracker VMs use TAP devices on the `tt0` bridge for networking -- Pause/resume is supported via the Firecracker API - -### QEMU / KVM - -QEMU images are **qcow2 disk image files** containing a bootable filesystem. - -**Image location:** -``` -/home/ttstack/images/ - └── qemu-test # Single qcow2 file (for file storage) -``` - -When using zvol storage, the image is written to a ZFS zvol exposed as a -raw block device at `/dev/zvol//`: -``` -/dev/zvol/tank/ttstack/images/qemu-test # Raw block device -``` - -**Creating manually:** - -```bash -# Create qcow2 image -qemu-img create -f qcow2 /home/ttstack/images/qemu-test 256M - -# Mount via NBD and populate -modprobe nbd max_part=8 -qemu-nbd --connect=/dev/nbd0 /home/ttstack/images/qemu-test -mkfs.ext4 /dev/nbd0 -mount /dev/nbd0 /tmp/mnt - -# Populate rootfs -mkdir -p /tmp/mnt/{bin,sbin,etc,proc,sys,dev} -# ... install packages or copy a static init binary ... - -umount /tmp/mnt -qemu-nbd --disconnect /dev/nbd0 -``` - -**Important notes:** -- QEMU uses KVM acceleration (`-enable-kvm`), so `/dev/kvm` must exist -- The disk is attached as virtio (`if=virtio`), so the guest kernel needs virtio drivers -- QEMU uses TAP devices on the `tt0` bridge for networking -- Stop/start uses the QEMU monitor (`stop`/`cont` commands) — the process stays alive - -## Storage Backend Considerations - -### File (default) - -Images are plain qcow2 files, filesystem-agnostic. On Linux, cloning uses -`cp --reflink=auto` (CoW if the filesystem supports it, full copy -otherwise). On FreeBSD, plain `cp -a` is used. - -```bash -# No special setup needed — just place images in image_dir -cp my-image.qcow2 /home/ttstack/images/qemu-test -``` - -### Zvol - -Images are stored as **ZFS zvols** — raw block devices exposed at -`/dev/zvol//`. Cloning uses `zfs snapshot` + `zfs clone` -— instant and space-efficient. - -```bash -# Setup -zpool create ttpool /dev/nvmeXnYpZ -zfs create ttpool/images -zfs create ttpool/runtime - -# Create image as a zvol -zfs create -V 10G ttpool/images/qemu-test -# Write image data to /dev/zvol/ttpool/images/qemu-test - -# Agent config -tt-agent --image-dir ttpool/images --runtime-dir ttpool/runtime --storage zvol -``` - -## Networking - -All VM engines (except Docker) use a shared network topology: - -``` -Guest VM ←→ TAP device ←→ tt0 bridge (10.10.0.1/16) ←→ NAT (nftables) ←→ Host -``` - -- Each VM gets a unique IP in the 10.10.0.0/16 range -- Port forwarding: host port → guest port via nftables DNAT (Linux) or PF rdr (FreeBSD) -- The `tt0` bridge and NAT table are created automatically by the agent - -Docker containers use Docker's native networking with `-p` port publishing. - -FreeBSD uses PF instead of nftables: - -``` -Guest VM ←→ TAP device ←→ tt0 bridge (10.10.0.1/16) ←→ NAT (PF) ←→ Host -``` - -## Creating a FreeBSD Test VM on Linux - -Running FreeBSD inside QEMU on a Linux host is useful for testing the -FreeBSD agent, controller, and CLI. This section documents the -non-interactive approach, including common pitfalls. - -### Recommended: mfsBSD Live ISO - -[mfsBSD](https://mfsbsd.vx.sk/) is a FreeBSD live CD that runs entirely -in RAM with SSH enabled out of the box. This is the fastest way to get -a working FreeBSD environment for build/test purposes. - -```bash -# Download mfsBSD SE (Special Edition includes base packages) -curl -sL https://mfsbsd.vx.sk/files/iso/14/amd64/mfsbsd-se-14.2-RELEASE-amd64.iso \ - -o /tmp/mfsbsd.iso - -# Create a work disk for persistent storage -qemu-img create -f qcow2 /tmp/fbsd-work.qcow2 20G - -# Boot the VM (8GB RAM recommended for compiling Rust) -qemu-system-x86_64 -enable-kvm -m 8192 -smp 4 \ - -drive file=/tmp/fbsd-work.qcow2,format=qcow2,if=virtio \ - -cdrom /tmp/mfsbsd.iso -boot d \ - -netdev user,id=net0,hostfwd=tcp::2222-:22 \ - -device virtio-net-pci,netdev=net0 \ - -vnc none -daemonize - -# SSH in (default root password: mfsroot) -sshpass -p 'mfsroot' ssh -p 2222 root@127.0.0.1 -``` - -### Installing FreeBSD to Disk from mfsBSD - -mfsBSD runs in RAM so installed packages are lost on reboot. For -persistent use, install FreeBSD to the work disk: - -```bash -# Inside the mfsBSD VM: -gpart create -s gpt vtbd0 -gpart add -t freebsd-boot -s 512k vtbd0 -gpart add -t freebsd-ufs -l rootfs vtbd0 -gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 vtbd0 -newfs -U /dev/vtbd0p2 - -# Mount and extract base + kernel -mkdir -p /rw/disk && mount /dev/vtbd0p2 /rw/disk -cd /rw/disk -fetch -o - https://download.freebsd.org/releases/amd64/14.3-RELEASE/base.txz | tar xf - -fetch -o - https://download.freebsd.org/releases/amd64/14.3-RELEASE/kernel.txz | tar xf - - -# Configure the installed system -echo '/dev/vtbd0p2 / ufs rw 1 1' > etc/fstab -cat > etc/rc.conf <<'RCEOF' -hostname="fbsd-test" -ifconfig_vtnet0="DHCP" -sshd_enable="YES" -sendmail_enable="NONE" -RCEOF -echo 'mfsroot' | chroot /rw/disk pw usermod root -h 0 -sed -i '' 's/^#PermitRootLogin .*/PermitRootLogin yes/' etc/ssh/sshd_config -echo 'nameserver 10.0.2.3' > etc/resolv.conf - -umount /rw/disk -``` - -Then reboot the VM without the `-cdrom` and `-boot d` flags to boot -from disk. - -### Common Pitfalls - -| Problem | Cause | Solution | -|---------|-------|----------| -| FreeBSD cloud images won't accept SSH | Root login disabled, no password auth, cloud-init issues | Use mfsBSD instead | -| `virt-customize` fails on FreeBSD images | libguestfs cannot write to UFS filesystems | Use mfsBSD or manual install | -| mfsBSD `pkg install` fails with "No error" | pkg 2.1.0 on mfsBSD 14.2 cannot handle zstd-packed repos | Install FreeBSD to disk for a newer pkg | -| `cp --reflink=auto` fails on FreeBSD | GNU option not available | Fixed in TTstack — FreeBSD uses `cp -a` | -| `ifconfig bridge create` names bridge `bridge0` | FreeBSD auto-assigns names | Fixed in TTstack — uses `ifconfig bridge create name tt0` | -| `ifconfig create` fails for custom names | Must specify type first | Fixed in TTstack — uses `ifconfig tap create name ` | -| Bhyve TAP name mismatch | Old code used `tap-{id}` instead of hashed name | Fixed in TTstack — uses `net::tap_name()` | -| OOM during Rust compilation on mfsBSD | mfsBSD runs in RAM; 4GB is insufficient | Use 8GB+ RAM or install to disk | diff --git a/docs/rest-api.md b/docs/rest-api.md deleted file mode 100644 index 2fb8a3b..0000000 --- a/docs/rest-api.md +++ /dev/null @@ -1,108 +0,0 @@ -# REST API Reference - -All `/api/*` endpoints require `Authorization: Bearer ` when the -controller is started with `--api-key`. The web dashboard (`/`) is always open. - -## Controller Endpoints - -| Method | Path | Description | -|--------|------|-------------| -| GET | `/` | Web dashboard (no auth required) | -| POST | `/api/hosts` | Register a host | -| GET | `/api/hosts` | List hosts | -| GET | `/api/hosts/{id}` | Host details | -| DELETE | `/api/hosts/{id}` | Remove host | -| POST | `/api/envs` | Create environment | -| GET | `/api/envs` | List environments | -| GET | `/api/envs/{id}` | Environment + VM details | -| DELETE | `/api/envs/{id}` | Destroy environment | -| POST | `/api/envs/{id}/stop` | Stop environment | -| POST | `/api/envs/{id}/start` | Start environment | -| GET | `/api/vms/{id}` | Single VM details | -| GET | `/api/images` | List images across fleet | -| GET | `/api/status` | Fleet-wide resource status | - -## Agent Endpoints - -| Method | Path | Description | -|--------|------|-------------| -| GET | `/api/info` | Host info and resources | -| GET | `/api/images` | Available images | -| POST | `/api/vms` | Create a VM | -| GET | `/api/vms` | List VMs | -| GET | `/api/vms/{id}` | VM details | -| DELETE | `/api/vms/{id}` | Destroy VM | -| POST | `/api/vms/{id}/stop` | Stop VM | -| POST | `/api/vms/{id}/start` | Start VM | - -## Examples - -### Register a host - -```bash -curl -X POST http://controller:9200/api/hosts \ - -H "Authorization: Bearer " \ - -H "Content-Type: application/json" \ - -d '{"addr": "10.0.0.2:9100"}' -``` - -### Create an environment - -```bash -curl -X POST http://controller:9200/api/envs \ - -H "Authorization: Bearer " \ - -H "Content-Type: application/json" \ - -d '{ - "id": "my-env", - "owner": "alice", - "ssh_keys": ["ssh-ed25519 AAAA... alice@laptop"], - "vms": [ - { - "image": "alpine-cloud", - "engine": "qemu", - "cpu": 2, - "mem": 2048, - "disk": 40960, - "ports": [80], - "deny_outgoing": false - } - ], - "lifetime": 21600 - }' -``` - -### Fleet status - -```bash -curl -H "Authorization: Bearer " http://controller:9200/api/status -``` - -## Request / Response Reference - -### CreateEnvReq (POST `/api/envs`) - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `id` | string | yes | Environment name | -| `owner` | string | no | Owner label | -| `ssh_keys` | string[] | yes | SSH public keys injected into all VMs (cloud-init `authorized_keys`) | -| `vms` | VmSpec[] | yes | List of VM specifications | -| `lifetime` | integer | no | Auto-expiry in seconds (default: 21600 = 6h) | - -### VmSpec (element of `vms` array) - -| Field | Type | Required | Description | -|-------|------|----------|-------------| -| `image` | string | yes | Base image name | -| `engine` | string | no | `qemu`, `firecracker`, `docker`, `bhyve`, `jail` (default: `qemu`) | -| `cpu` | integer | no | vCPUs (default: 2) | -| `mem` | integer | no | Memory in MiB (default: 1024) | -| `disk` | integer | no | Disk in MiB (default: 40960) | -| `ports` | integer[] | no | Guest ports to expose; port 22 is always auto-included | -| `deny_outgoing` | boolean | no | Block outbound traffic (default: false) | - -### Storage field (agent `/api/info`) - -The `storage` field in host info reports the backend type: -- `"file"` — plain qcow2 files, filesystem-agnostic (aliases: `"raw"`) -- `"zvol"` — ZFS zvol raw block devices (aliases: `"zfs"`) diff --git a/documents/arch_design.md b/documents/arch_design.md new file mode 100644 index 0000000..a98b4ba --- /dev/null +++ b/documents/arch_design.md @@ -0,0 +1,72 @@ +# System Design + +## 架构说明 + +### 上层逻辑 + +TT 中的基本管理单位为 “环境(ENV)”, 如下以时序图的方式展现一个 “环境” 的创建过程. + +```mermaid +sequenceDiagram + autonumber + + participant C as Client + participant P as Proxy + participant S as Server[s] + participant R as Core + + C->>P: 创建 VM 的请求 + par 分发请求 + P-->>S: 计算可用资源, 并行分发 + end + + S->>R: 调用 Core 创建 VM + Note right of S: 配置网络及生命周期 + R->>S: return + + par 返回结果 + S->>P: 异步返回 + end + + P->>C: 聚合各 Server 的结果 + + loop 资源管理 + R-->>R: 定时清理过期的 VM + end +``` + +### Core 内部实现 + +#### On Linux + +```mermaid +sequenceDiagram + autonumber + + participant C as Core + participant K as Kernel + participant Q as Qemu/FireCracker + participant N as Nftables + + C->>K: 创建 PID NS 与 CGROUP + C->>Q: 增量(COW)创建运行时镜像 + C->>N: 使用 Nftables 的哈希表结构管理 NAT 规则 +``` + +## Why NOT + +### Why NOT K8S + +K8S 主要用于调度容器, 不适于对隔离性要求较高的场景. + +### Why NOT OpenStack + +OpenStack 太过复杂, 需要专门的团队维护, 成本太高. + +### Why NOT Ansible + +Ansible 只是一个批量管理工具, 不具备虚拟方案的管理与调度功能. + +### Why NOT Libvirt + +Libvirt 在安装系统及远程管理方面非常便捷, 但不具备自动化调度的能力. 当前 TT 系统使用 Libvirt 做为基础系统镜像的安装工具. diff --git a/documents/code_about.md b/documents/code_about.md new file mode 100644 index 0000000..d11ba94 --- /dev/null +++ b/documents/code_about.md @@ -0,0 +1,71 @@ +# tt + +tt 主要代码使用 Rust 编写. + +Rust 是一门风格**紧凑**、运行**高效**的现代开发语言, 以**最少的代码量**实现**最高的性能**, 打破了几十年来的旷世难题: 静态语言与动态语言两者的优势不能兼得. + +## 项目结构 + +``` +$ (git)-[master]-% tree -F -I 'target' -L 1 +. +├── Cargo.toml # 项目配置文件 +├── README.md # 项目主文档 +├── documents/ # 项目详细文档 +├── src/core/ # 服务端的核心逻辑实现 +├── src/core_def/ # 从 core 模块中提取出的通用定义, 供 server 模块使用 +├── src/server/ # 后端 Server 的代码实现, 可独立运行, 也可挂靠在 Proxy 之后 +├── src/server_def/ # 从 server 模块中提取出的通用定义, 供 proxy 模块使用 +├── src/proxy/ # 分布式架构后端, 负责统筹调度多个 Server 的资源 +├── src/rexec/ # 一个轻量级的"远程命令执行和文件转输"方案 +├── src/client/ # 客户端 tt 命令的代码实现 +├── tools/ # 外围脚本工具 +└── ... +``` + +## 代码规模 + +``` +$ (git)-[master]-% find . -type f \ + | grep -Ev 'target|tools/(.*kernel_config|firecracker)|\.(git|lock)' \ + | xargs wc -l \ + | grep -Ev '^ +[0-9]{1,2} ' + + 108 ./Makefile + 194 ./README.md + 482 ./documents/user_guide.md + 169 ./src/rexec/tests/integration.rs + 117 ./src/rexec/src/bin/cli.rs + 149 ./src/rexec/src/client.rs + 272 ./src/rexec/src/server.rs + 164 ./src/rexec/src/common.rs + 1220 ./src/core/src/def.rs + 293 ./src/core/src/linux/nat/mod.rs + 205 ./src/core/src/linux/mod.rs + 150 ./src/core/src/linux/vm/cgroup.rs + 357 ./src/core/src/linux/vm/engine/qemu.rs + 245 ./src/core/src/linux/vm/engine/firecracker/suitable_env.rs + 312 ./src/proxy/tests/knead/mod.rs + 220 ./src/proxy/tests/standalone/mod.rs + 179 ./src/proxy/tests/env/mod.rs + 151 ./src/proxy/src/util.rs + 102 ./src/proxy/src/def.rs + 145 ./src/proxy/src/hdr/add_env.rs + 424 ./src/proxy/src/hdr/mod.rs + 265 ./src/proxy/src/lib.rs + 198 ./src/core_def/src/lib.rs + 267 ./src/server/tests/knead/mod.rs + 193 ./src/server/tests/standalone/mod.rs + 107 ./src/server/tests/env/mod.rs + 100 ./src/server/src/bin/ttserver.rs + 329 ./src/server/src/hdr/mod.rs + 120 ./src/server/src/lib.rs + 231 ./src/server_def/src/lib.rs + 555 ./src/client/src/cmd_line.rs + 157 ./src/client/src/ops/env/update.rs + 253 ./src/client/src/ops/env/run/mod.rs + 114 ./src/client/src/ops/env/run/ssh.rs + 131 ./src/client/src/ops/mod.rs + 149 ./src/client/src/cfg_file.rs + 12034 total +``` diff --git a/documents/firecracker_kernel_notes.md b/documents/firecracker_kernel_notes.md new file mode 100644 index 0000000..4ab7ba3 --- /dev/null +++ b/documents/firecracker_kernel_notes.md @@ -0,0 +1,34 @@ +# Kernel + +## Configs MUST be Built In + +#### file system + +- CONFIG_EXT4_FS + +#### virtio basic + +- CONFIG_VIRTIO_PCI +- CONFIG_VIRTIO_INPUT +- CONFIG_VIRTIO_MMIO +- CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES + +#### virtio block device + +- CONFIG_VIRTIO_BLK + +#### virtio network device + +- CONFIG_VSOCKETS +- CONFIG_VIRTIO_VSOCKETS +- CONFIG_VIRTIO_NET + +#### make script[s] exectuable + +- BINFMT_MISC + +## Configs MUST be disabled + +#### accept stripped modular[s] + +- CONFIG_MODULE_SIG diff --git a/documents/roadmap.md b/documents/roadmap.md new file mode 100644 index 0000000..96d7b46 --- /dev/null +++ b/documents/roadmap.md @@ -0,0 +1,41 @@ +# 开发路线 (RoadMap) + +## v0.1.x + +- [Y] 单点架构, 基本功能可用 + +## v0.2.x + +- [Y] 分布式架构, 核心功能可用 + +## v0.3.x + +- [Y] v0.3.1: 优化资源调度算法 +- [Y] v0.3.1: 支持服务重启后恢复已有的 ENV +- [-] 优化系统布署流程, 提供配置的自动化工具集 +- [-] 优化基础镜像制作流程, 提供配置的自动化工具集 +- [-] 提高核心模块的测试覆盖率 +- [-] 提供视频演示教程 + +## v0.4.x + +- [-] 优化命令行客户端的用户提示信息 + - 当前显示给用户的是开发端的信息 +- [-] 提供前端界面 + - 初步规划是基于 Yew/Seed 等 Rust 前端框架实现 +- [-] 完善英文文档 + +## v0.5.x + +- [-] 优化前端界面实现, 增强特性 + - 如: Web Terminal 等功能 +- [-] 提供一个实时体验的网址 + +## v0.6.x + +- [-] 支持将运行时镜像保存为基础镜像(模板镜像) +- [-] 支持对已创建的 ENV 打快照(多级快照如何高效管理?) + +## v0.7.x + +- [-] 待定... diff --git a/documents/system_admin.md b/documents/system_admin.md new file mode 100644 index 0000000..828a2b0 --- /dev/null +++ b/documents/system_admin.md @@ -0,0 +1,68 @@ +# TT 系统管理员指南 + +Guide for system admin. + +## 环境准备 + +> If use zfs, should do this: `zfs create -V 1M zroot/tt/__sample__`. + +镜像相关: +- [Linux, MUST] `systemctl disable NetworkManager` +- [Linux, Optional] `systemctl disable firewalld` +- [Linux, Optional] `sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config` +- 保证本项目定制的 "[rc.local](../tools/images/linux_vm/rc.local)" 文件开机启动 +- [ FireCracker ] 内核模块只能 `strip --strip-debug`, strip 过度会导致无法载入 +- [ FireCracker ] 关闭内核的模块签名校验功能, 因为签名信息存在于已被 strip 的 debug 信息中 +- [ FireCracker ] `firecracker` 二进制文件要使用 `musl-libc` 编译 + - 由于 `Rust` 动态链接 `glibc` , 故运行时存在更多的不确定性因素, 参见: [issue #2044](https://github.com/firecracker-microvm/firecracker/issues/2044) + +#### On Linux + +> 使用 Qemu\FireCracker 做为 VM 引擎, 使用 Nftables 做 NAT 端口转发. +> +> - `firecracker` 程序路径必须是 `/usr/sbin/firecracker` + +环境配置: + +- `modprobe tun vhost_net` + +组件安装 + +- `qemu` +- `nftables` +- `ZOL: zfs on linux` + +## ttserver 配置 + +```shell +USAGE: + ttserver [FLAGS] [OPTIONS] + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + --cpu-total 可以使用的 CPU 核心总数. + --disk-total 可以使用的磁盘总量, 单位: MB. + --image-path 镜像存放路径. + --log-path 日志存储路径. + --mem-total 可以使用的内存总量, 单位: MB. + --serv-addr 服务监听地址. + --serv-port 服务监听端口. +``` + +## ttproxy 配置 + +```shell +USAGE: + ttproxy [OPTIONS] + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +OPTIONS: + --proxy-addr ttproxy 地址, eg: 127.0.0.1:19527. + --server-set ... ttserver 地址, eg: 127.0.0.1:9527,10.10.10.101:9527. +``` diff --git a/documents/user_guide.md b/documents/user_guide.md new file mode 100644 index 0000000..b184ca1 --- /dev/null +++ b/documents/user_guide.md @@ -0,0 +1,482 @@ +# tt client + +用户用来与 TT 服务端进行交互的工具. + +> **仅支持 Linux 与 MacOS** + +## 安装 + +```shell +sudo make install +chmod +x /usr/local/bin/tt +``` + +## 使用 + +```shell +USAGE: + tt [SUBCOMMAND] + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +SUBCOMMANDS: + config + env + help Prints this message or the help of the given subcommand(s) + status +``` + +#### 配置 tt 服务端的地址 + +##### 命令参数 + +> ```shell +> tt-config +> +> USAGE: +> tt config [OPTIONS] +> +> FLAGS: +> -h, --help Prints help information +> -V, --version Prints version information +> +> OPTIONS: +> -n, --client-id 客户端别名. +> -a, --server-addr 服务端的监听地址. +> -p, --server-port 服务端的监听端口. +> ``` + +##### 示例 + +> ```shell +> tt config --server-addr=10.10.10.22 [--server-port=9527] +> ``` + +#### 创建环境 + +##### 命令参数 + +> ```shell +> tt-env-add +> +> USAGE: +> tt env add [FLAGS] [OPTIONS] +> tt env add [FLAGS] [OPTIONS] -- +> +> FLAGS: +> -n, --deny-outgoing 禁止虚拟机对外连网. +> -h, --help Prints help information +> --same-uuid 所有虚拟机都使用同一个 UUID. +> -V, --version Prints version information +> +> OPTIONS: +> -C, --cpu-num 虚拟机的 CPU 核心数量. +> -D, --disk-size 虚拟机的磁盘容量, 单位: MB. +> -d, --dup-each 每种虚拟机类型启动的实例数量. +> -l, --life-time