module;

#include <cstdlib> // getenv

// mcpp.toolchain.stdmod — pre-build the `import std` BMI and cache it.

//

// GCC 15 flow (from docs/11-gcc15-cookbook.md §2):

// g++ -std=c++23 -fmodules -Og -c <std.cc> -o std.o

// ⇒ produces gcm.cache/std.gcm + std.o

//

// Clang/libc++ flow:

// clang++ -std=c++23 --precompile <std.cppm> -o pcm.cache/std.pcm

// clang++ -std=c++23 pcm.cache/std.pcm -c -o std.o

//

// We invoke the compiler in a dedicated cache directory so the produced

// BMI is owned by mcpp and reused across every build with the same std identity.

//

// Output layout:

// <cache_root>/std/<std_identity_key>/

// gcm.cache/std.gcm ← GCC BMI

// pcm.cache/std.pcm ← Clang BMI

// std.o ← linked into final binaries

//

// The directory used to be named by the WHOLE-PROJECT fingerprint, which folds

// in the root package's name, version and flags — none of which reach the std

// module's compile commands. The metadata written next to the artifacts

// (metadata_for below) has always been the correct identity and has always been

// what validates a hit; only the directory name was wrong. On one machine that

// cost 1014 directories holding 15 distinct identities: 16.1 GB where ~0.5 GB

// was needed, and a `mcpp version bump` re-precompiled std from scratch.

export module mcpp.toolchain.stdmod;

import std;

import mcpp.home;

import mcpp.libs.json;

import mcpp.platform;

import mcpp.toolchain.clang;

import mcpp.toolchain.detect;

import mcpp.toolchain.fingerprint;

import mcpp.toolchain.gcc;

import mcpp.toolchain.hostflags;

import mcpp.toolchain.linkmodel;

import mcpp.toolchain.msvc;

export namespace mcpp::toolchain {

struct StdModule {

std::filesystem::path cacheDir; // <cache_root>/std/<key>/

std::filesystem::path bmiPath; // <cacheDir>/gcm.cache/std.gcm

std::filesystem::path objectPath; // <cacheDir>/std.o

std::filesystem::path compatBmiPath; // <cacheDir>/pcm.cache/std.compat.pcm

std::filesystem::path compatObjectPath; // <cacheDir>/std.compat.o

};

struct StdModError { std::string message; };

std::filesystem::path default_cache_root();

// Build std module if not already cached. Returns paths to BMI + object.

// `macos_deployment_target` is the RESOLVED value from

// platform::macos::deployment_target() — it must match what flags.cppm

// emits for normal TUs, or the produced std.pcm targets a different

// arm64-apple-macosxNN triple than the code importing it.

std::expected<StdModule, StdModError> ensure_built(

const Toolchain& tc,

std::string_view cpp_standard,

std::string_view cpp_standard_flag,

std::string_view macos_deployment_target = {},

const std::filesystem::path& cache_root = default_cache_root(),

// MSVC only: the CRT model (`/MT` or `/MD`) the project's TUs use. Same

// contract as `macos_deployment_target` above and for the same reason —

// cl bakes `_MSVC_MT` / `_MSVC_MD` into the module, so a std built with the

// other one makes every importing TU fail in the ucrt headers. It also

// enters `std_build_commands`, which is part of the cache identity, so two

// CRT models cannot share a cache directory.

std::string_view msvc_crt_flag = {});

} // namespace mcpp::toolchain

namespace mcpp::toolchain {

namespace {

std::expected<std::string, StdModError> run_capture_command(

const std::string& cmd,

const std::vector<std::pair<std::string, std::string>>& env) {

auto r = env.empty()

? mcpp::platform::process::capture(cmd)

: mcpp::platform::process::capture_with_env(cmd, env);

if (r.exit_code != 0) {

// Include the command: its --sysroot/-isystem flags are the first

// thing needed to diagnose header-resolution failures.

return std::unexpected(StdModError{

std::format("std module precompile failed (rc={}):\n{}\ncommand: {}",

r.exit_code, r.output, cmd)});

}

return r.output;

}

std::filesystem::path metadata_path(const std::filesystem::path& cacheDir) {

return cacheDir / "std-module.json";

}

// The directory segment used while deriving the identity, so the build commands

// folded into it do not contain the identity they are about to produce.

constexpr std::string_view kStdKeyPlaceholder = "@std-key@";

// The std module's cache directory name. Derived from the metadata that already

// defines std identity (metadata_for below: compiler, version, driver identity,

// target triple, stdlib, standard + flag, source hashes, build commands) — the

// same object metadata_matches validates a hit against. Naming the directory

// after the whole-project fingerprint instead is what let 15 real identities

// occupy 1014 directories.

//

// Format compatibility is carried by the `build-cache/v1` path segment above

// one, not by an epoch field: a std BMI's validity depends on the toolchain,

// never on mcpp's cache bookkeeping.

std::string std_identity_key(const nlohmann::json& normalized) {

return hash_string("mcpp-std-key-v1\x1f" + normalized.dump());

}

nlohmann::json metadata_for(const Toolchain& tc,

std::string_view cppStandard,

std::string_view cppStandardFlag,

const std::vector<std::string>& stdCommands,

const std::vector<std::string>& compatCommands) {

nlohmann::json j;

j["schema"] = 1;

j["compiler"] = std::string(tc.compiler_name());

j["compiler_version"] = tc.version;

j["driver_identity"] = tc.driverIdent.empty()

? (tc.binaryPath.empty() ? "" : hash_file(tc.binaryPath))

: hash_string(tc.driverIdent);

j["target_triple"] = tc.targetTriple;

j["stdlib"] = tc.stdlibId;

j["stdlib_version"] = tc.stdlibVersion;

j["cpp_standard"] = std::string(cppStandard);

j["std_flag"] = std::string(cppStandardFlag);

j["std_module_source"] = tc.stdModuleSource.generic_string();

j["std_module_source_hash"] = hash_file(tc.stdModuleSource);

j["std_compat_source"] = tc.stdCompatSource.generic_string();

j["std_compat_source_hash"] = tc.stdCompatSource.empty() ? "" : hash_file(tc.stdCompatSource);

j["std_build_commands"] = stdCommands;

j["std_compat_build_commands"] = compatCommands;

return j;

}

bool metadata_matches(const std::filesystem::path& path, const nlohmann::json& expected) {

std::ifstream is(path);

if (!is) return false;

nlohmann::json actual;

try {

is >> actual;

} catch (...) {

return false;

}

static constexpr std::array<std::string_view, 14> keys = {

"schema",

"compiler",

"compiler_version",

"driver_identity",

"target_triple",

"stdlib",

"stdlib_version",

"cpp_standard",

"std_flag",

"std_module_source",

"std_module_source_hash",

"std_compat_source",

"std_compat_source_hash",

"std_build_commands",

};

for (auto key : keys) {

auto k = std::string(key);

if (!actual.contains(k) || actual[k] != expected[k]) return false;

}

return actual.value("std_compat_build_commands", nlohmann::json::array())

== expected["std_compat_build_commands"];

}

std::expected<void, StdModError> write_metadata(const std::filesystem::path& path,

const nlohmann::json& metadata) {

std::ofstream os(path, std::ios::binary);

if (!os) {

return std::unexpected(StdModError{

std::format("cannot write std module metadata '{}'", path.string())});

}

os << metadata.dump(2) << "\n";

if (!os) {

return std::unexpected(StdModError{

std::format("failed while writing std module metadata '{}'", path.string())});

}

return {};

}

// Toolchain-declared env (MSVC's INCLUDE/LIB/PATH) applies to every std

// module build command; empty for GCC/Clang (their LD_LIBRARY_PATH need is

// carried as an in-command `env` prefix on POSIX for now).

std::expected<std::string, StdModError> run_commands(

const std::vector<std::string>& commands, const Toolchain& tc) {

std::vector<std::pair<std::string, std::string>> env;

for (auto& ev : tc.envOverrides) env.emplace_back(ev.key, ev.value);

std::string out;

for (auto const& cmd : commands) {

if (auto r = run_capture_command(cmd, env); !r) return std::unexpected(r.error());

else out += *r;

}

return out;

}

} // namespace

std::filesystem::path default_cache_root() {

// Single resolver (#311). This used to be a private copy of the home

// resolution that predated Windows' USERPROFILE branch and self-contained

// installs, so the std BMI cache could land in the *current working

// directory* (`.mcpp-bmi/`) while dep BMIs went to $MCPP_HOME/bmi.

return mcpp::home::cache_root();

}

std::expected<StdModule, StdModError> ensure_built(

const Toolchain& tc,

std::string_view cpp_standard,

std::string_view cpp_standard_flag,

std::string_view macos_deployment_target,

const std::filesystem::path& cache_root,

std::string_view msvc_crt_flag)

{

if (tc.stdModuleSource.empty()) {

return std::unexpected(StdModError{

"toolchain has no std module source (import std unsupported on this compiler)"});

}

const bool isMsvc = tc.compiler == CompilerId::MSVC;

StdModule sm;

// Build sysroot + include flags for std module precompilation, derived

// from the shared toolchain link model (same resolver as flags.cppm —

// identical flags also keep the std_build_commands cache key honest).

// Std module precompilation only needs compile flags (no linker flags),

// so --no-default-config is safe here on all platforms.

const PathEscape shellEsc = [](const std::filesystem::path& p) {

return std::format("'{}'", p.string());

};

// The shared producer (mcpp.toolchain.hostflags) — the same assembly

// flags.cppm and the build.mcpp host compile use. This block used to

// hand-write the clang cfg bypass, which is how it could drift from the

// other two.

HostFlagOptions hopt;

hopt.cfgBypass = HostFlagOptions::CfgBypass::Always;

hopt.clangStdlibSelect = true;

std::string sysroot_flag =

render_tokens(host_compile_tokens(tc, hopt, shellEsc));

// Deployment target appended here rather than passed to the producer

// ONLY to keep this command string byte-identical to what earlier

// releases emitted: the string is part of the std cache identity

// (std_build_commands feeds the cache directory name), so reordering a

// flag would invalidate every user's std BMIs for no behavioural gain.

// The VALUE still comes from the one resolver

// (platform::macos::deployment_target) that flags.cppm and

// build_program.cppm read — only its position is local.

if (!macos_deployment_target.empty()) {

sysroot_flag += std::format(" -mmacosx-version-min={}",

macos_deployment_target);

}

// All three providers expose the same command-sequence shape (A5 backend

// surface normalization) — no per-compiler arity branching here.

//

// Everything downstream of the cache directory is derived here, because the

// directory NAME is derived from the metadata and the metadata contains the

// build commands, which contain the directory. Resolving that circularity

// by hand: run this once against a placeholder directory to obtain the

// identity (that is what "normalized" means — the commands then name a

// literal placeholder segment instead of a key-specific path), hash it,

// then run it again against the real directory to get what goes on disk.

struct Derived {

std::filesystem::path bmiPath;

std::filesystem::path objectPath;

std::vector<std::string> stdCommands;

std::vector<std::string> compatCommands;

nlohmann::json metadata;

};

auto derive = [&](const std::filesystem::path& cacheDir) {

Derived d;

d.bmiPath = isMsvc ? mcpp::toolchain::msvc::std_bmi_path(cacheDir)

: is_clang(tc)

? mcpp::toolchain::clang::std_bmi_path(cacheDir)

: mcpp::toolchain::gcc::std_bmi_path(cacheDir);

d.objectPath = cacheDir / (isMsvc ? "std.obj" : "std.o");

d.stdCommands =

isMsvc ? mcpp::toolchain::msvc::std_module_build_commands(

tc, cacheDir, cpp_standard_flag, msvc_crt_flag)

: is_clang(tc)

? mcpp::toolchain::clang::std_module_build_commands(

tc, cacheDir, d.bmiPath, sysroot_flag, cpp_standard_flag)

: mcpp::toolchain::gcc::std_module_build_commands(

tc, cacheDir, sysroot_flag, cpp_standard_flag);

if (!tc.stdCompatSource.empty()) {

if (isMsvc) {

d.compatCommands = mcpp::toolchain::msvc::std_compat_build_commands(

tc, cacheDir, cpp_standard_flag, msvc_crt_flag);

} else if (is_clang(tc)) {

auto compatBmi = mcpp::toolchain::clang::std_compat_bmi_path(cacheDir);

d.compatCommands = mcpp::toolchain::clang::std_compat_build_commands(

tc, cacheDir, compatBmi, d.bmiPath, sysroot_flag, cpp_standard_flag);

}

}

d.metadata = metadata_for(tc, cpp_standard, cpp_standard_flag,

d.stdCommands, d.compatCommands);

return d;

};

const auto stdRoot = cache_root / "std";

auto normalized = derive(stdRoot / kStdKeyPlaceholder).metadata;

auto identity = std_identity_key(normalized);

sm.cacheDir = stdRoot / identity;

auto derived = derive(sm.cacheDir);

sm.bmiPath = derived.bmiPath;

sm.objectPath = derived.objectPath;

const auto& stdCommands = derived.stdCommands;

const auto& compatCommands = derived.compatCommands;

const auto& metadata = derived.metadata;

auto metaPath = metadata_path(sm.cacheDir);

bool std_cached = std::filesystem::exists(sm.bmiPath)

&& std::filesystem::exists(sm.objectPath)

&& metadata_matches(metaPath, metadata);

bool rebuiltStd = false;

if (!std_cached) {

std::error_code ec;

std::filesystem::create_directories(sm.bmiPath.parent_path(), ec);

if (ec) return std::unexpected(StdModError{

std::format("cannot create '{}': {}", sm.bmiPath.parent_path().string(), ec.message())});

auto out = run_commands(stdCommands, tc);

if (!out) return std::unexpected(out.error());

if (!std::filesystem::exists(sm.bmiPath)) {

return std::unexpected(StdModError{

std::format("expected BMI at '{}' but it wasn't produced; output:\n{}",

sm.bmiPath.string(), *out)});

}

rebuiltStd = true;

}

// Build std.compat after std (std.compat imports std; Clang + MSVC).

if (!compatCommands.empty()) {

auto compatBmi = isMsvc

? mcpp::toolchain::msvc::std_compat_bmi_path(sm.cacheDir)

: mcpp::toolchain::clang::std_compat_bmi_path(sm.cacheDir);

if (rebuiltStd || !std::filesystem::exists(compatBmi)

|| !metadata_matches(metaPath, metadata)) {

if (auto out = run_commands(compatCommands, tc); !out) {

return std::unexpected(out.error());

}

}

sm.compatBmiPath = compatBmi;

sm.compatObjectPath = sm.cacheDir / (isMsvc ? "std.compat.obj" : "std.compat.o");

}

if (auto r = write_metadata(metaPath, metadata); !r) {

return std::unexpected(r.error());

}

return sm;

}

} // namespace mcpp::toolchain