diff --git a/.github/workflows/release_trigger.yml b/.github/workflows/release_trigger.yml new file mode 100644 index 0000000..cfa7986 --- /dev/null +++ b/.github/workflows/release_trigger.yml @@ -0,0 +1,28 @@ +name: 🚀 Release Trigger + +on: + workflow_dispatch: + inputs: + draft: + type: boolean + description: "Create Release Draft" + required: false + default: true + release_overwrite: + type: string + description: "Set Version Release Tag" + required: false + +jobs: + call-release-trigger: + uses: ynput/ops-repo-automation/.github/workflows/release_trigger.yml@main + + with: + draft: ${{ inputs.draft }} + release_overwrite: ${{ inputs.release_overwrite }} + build_package: false + + secrets: + token: ${{ secrets.YNPUT_BOT_TOKEN }} + email: ${{ secrets.CI_EMAIL }} + user: ${{ secrets.CI_USER }} \ No newline at end of file diff --git a/.github/workflows/validate_pr_labels.yml b/.github/workflows/validate_pr_labels.yml new file mode 100644 index 0000000..30cc7f4 --- /dev/null +++ b/.github/workflows/validate_pr_labels.yml @@ -0,0 +1,18 @@ +name: 🔎 Validate PR Labels +on: + pull_request: + types: + - opened + - edited + - labeled + - unlabeled + +jobs: + validate-type-label: + uses: ynput/ops-repo-automation/.github/workflows/validate_pr_labels.yml@main + with: + repo: "${{ github.repository }}" + pull_request_number: ${{ github.event.pull_request.number }} + query_prefix: "type: " + secrets: + token: ${{ secrets.YNPUT_BOT_TOKEN }} \ No newline at end of file diff --git a/ext/ayon-cpp-dev-tools b/ext/ayon-cpp-dev-tools index 9f647c0..3ad2d04 160000 --- a/ext/ayon-cpp-dev-tools +++ b/ext/ayon-cpp-dev-tools @@ -1 +1 @@ -Subproject commit 9f647c0eaef710d6e5f8564f65a6091d19acc791 +Subproject commit 3ad2d041441a1acdb29ecf81a1f7d20750f8ee2d diff --git a/src/AyonCppApi/AyonCppApi.cpp b/src/AyonCppApi/AyonCppApi.cpp index d2284a5..7d6b158 100644 --- a/src/AyonCppApi/AyonCppApi.cpp +++ b/src/AyonCppApi/AyonCppApi.cpp @@ -109,7 +109,7 @@ AyonApi::AyonApi(const std::optional &logFilePos, // ----------- Resolve Log Path std::filesystem::path logPath; - if (logFilePos.has_value()) { + if (logFilePos && !logFilePos->empty()) { try { std::filesystem::path inPath(logFilePos.value()); std::cout << "Input log path: " << inPath << std::endl; @@ -149,10 +149,18 @@ AyonApi::AyonApi(const std::optional &logFilePos, m_log = std::shared_ptr(&loggerRef, [](AyonLogger*){}); m_log->registerLoggingKey("AyonApi"); - m_log->setLogLevelInfo(); + m_log->setLogLevelFromEnv(); m_log->info(m_log->key("AyonApi"), "Init AyonServer httplib::Client"); m_ayonServer = std::make_unique(m_serverUrl); + // Reuse the TCP/TLS connection across resolves instead of a fresh handshake per + // request. Over a WAN link the handshake dominates per-call latency, so this is + // a large win for the serial resolve path and the prewarm batched resolves. + // Gated so the keep-alive contribution can be benchmarked: set + // AYON_RESOLVER_NO_KEEPALIVE=1 to fall back to a fresh handshake per request. + if (!std::getenv("AYON_RESOLVER_NO_KEEPALIVE")) { + m_ayonServer->set_keep_alive(true); + } m_log->info(m_log->key("AyonApi"), "After creating httplib::Client - {}", m_serverUrl); if (isSSL()) { @@ -160,6 +168,7 @@ AyonApi::AyonApi(const std::optional &logFilePos, if (!ayonSSLPath.empty()) { m_log->info(m_log->key("AyonApi"), "Using AYON_SSL_CERT_PATH: {}", ayonSSLPath); m_ayonServer->set_ca_cert_path(ayonSSLPath.c_str()); + m_caCertPath = ayonSSLPath; } else { m_log->warn(m_log->key("AyonApi"), "No AYON_SSL_CERT_PATH set, trying to get OpenSSL dir"); try { @@ -192,9 +201,15 @@ AyonApi::AyonApi(const std::optional &logFilePos, m_log->info(m_log->key("AyonApi"), "Status code: {}", res->status); m_headers = { - {"X-Api-Key", m_authKey}, - {"X-ayon-site-id", m_siteId} + {"X-Api-Key", m_authKey} }; + // Only advertise a site id when we actually have one. A service-account + // setup (or any machine without a registered AYON site) has no valid + // site, and the server returns 400 "Invalid site id" if the header is + // present but empty/unknown. Omitting it lets resolution proceed. + if (!m_siteId.empty()) { + m_headers.emplace("X-ayon-site-id", m_siteId); + } auto resMe = m_ayonServer->Get("/api/users/me", m_headers); if (resMe && resMe->status != 200) { @@ -549,6 +564,70 @@ AyonApi::batchResolvePath(std::vector &uriPaths) { return assetIdentGrp; }; +std::unordered_map +AyonApi::batchResolvePathSerial(const std::vector &uriPaths) { + PerfTimer("AyonApi::batchResolvePathSerial"); + m_log->info(m_log->key("AyonApi"), "AyonApi::batchResolvePathSerial({} uris)", uriPaths.size()); + + std::unordered_map assetIdentGrp; + if (uriPaths.empty()) { + return assetIdentGrp; + } + + // Drop empties up front so chunk boundaries are stable and we never POST blank uris. + std::vector cleanUris; + cleanUris.reserve(uriPaths.size()); + for (const auto &uri: uriPaths) { + if (!uri.empty()) { + cleanUris.push_back(uri); + } + } + if (cleanUris.empty()) { + return assetIdentGrp; + } + + const std::string endPoint + = m_pathOnlyResolution ? m_uriResolverEndpoint + m_uriResolverEndpointPathOnlyVar : m_uriResolverEndpoint; + + // Split large frontiers into capped chunks (see m_maxSerialBatchSize). Each chunk is its + // own POST over the keep-alive client, so the server processes and releases a bounded + // batch at a time. For typical frontiers (<= m_maxSerialBatchSize) this is a single + // request, identical to the un-chunked path. + const size_t chunkSize = m_maxSerialBatchSize; + for (size_t start = 0; start < cleanUris.size(); start += chunkSize) { + const size_t end = std::min(start + chunkSize, cleanUris.size()); + + nlohmann::json uriArray = nlohmann::json::array(); + for (size_t i = start; i < end; ++i) { + uriArray.push_back(cleanUris[i]); + } + nlohmann::json jsonPayload = {{"resolveRoots", false}, {"uris", uriArray}}; + std::string payload = jsonPayload.dump(); + + std::string rawResponse; + { + std::lock_guard lock(m_ayonServerMutex); + rawResponse = serialCorePost(endPoint, m_headers, payload, 200); + } + if (rawResponse.empty()) { + m_log->warn("AyonApi::batchResolvePathSerial empty response for chunk [{}, {})", start, end); + continue; + } + + try { + nlohmann::json responseArray = nlohmann::json::parse(rawResponse); + for (const auto &assetRaw: responseArray) { + assetIdentGrp.emplace(getAssetIdent(assetRaw)); + } + } + catch (const nlohmann::json::exception &e) { + m_log->error("AyonApi::batchResolvePathSerial JSON parse failed: {}", e.what()); + } + } + + return assetIdentGrp; +}; + // TODO make it so that hero version is chosen if available std::pair AyonApi::getAssetIdent(const nlohmann::json &uriResolverResponse) { @@ -603,6 +682,25 @@ AyonApi::serialCorePost(const std::string &endPoint, while (retries <= m_maxCallRetries) { try { response = m_ayonServer->Post(endPoint, headers, Payload, "application/json"); + if (!response) { + auto err = response.error(); + m_log->warn("AyonApi::serialCorePost response is null: {}", httplib::to_string(err)); + if (err == httplib::Error::SSLServerVerification) { + if (m_caCertPath.empty() || !std::filesystem::exists(m_caCertPath) || + !std::filesystem::is_regular_file(m_caCertPath)) { + m_log->error("AyonApi::serialCorePost SSL verification failed and cert path is invalid: '{}' - not retrying", m_caCertPath); + return ""; + } + m_log->warn("AyonApi::serialCorePost SSL verification failed with a valid cert path present - rebuilding client and retrying"); + m_ayonServer = std::make_unique(m_serverUrl); + m_ayonServer->set_keep_alive(true); + m_ayonServer->set_ca_cert_path(m_caCertPath.c_str()); + m_ayonServer->enable_server_certificate_verification(true); + } + retries++; + std::this_thread::sleep_for(std::chrono::milliseconds(m_retryWait)); + continue; + } responseStatus = response->status; retries++; @@ -775,6 +873,7 @@ AyonApi::setSSL() { if (std::filesystem::exists(envCertFile)) { m_log->info("Using cert based on env variable (SSL_CERT_FILE): {}", envCertFile); m_ayonServer->set_ca_cert_path(envCertFile); + m_caCertPath = envCertFile; return; } } @@ -786,6 +885,7 @@ AyonApi::setSSL() { if (std::filesystem::exists(certFileCLI)) { m_log->info("Using cert based on CLI var: {}", certFileCLI); m_ayonServer->set_ca_cert_path(certFileCLI.c_str()); + m_caCertPath = certFileCLI; return; } @@ -796,6 +896,7 @@ AyonApi::setSSL() { if (std::filesystem::exists(certFileSSLEAY)) { m_log->info("Using cert based on SSLEAY_DIR: {}", certFileSSLEAY); m_ayonServer->set_ca_cert_path(certFileSSLEAY.c_str()); + m_caCertPath = certFileSSLEAY; return; } @@ -835,6 +936,7 @@ AyonApi::setSSL() { if (std::filesystem::exists(certPath)) { m_log->info("Using bundled certificate (via library path): {}", certPath); m_ayonServer->set_ca_cert_path(certPath.c_str()); + m_caCertPath = certPath; return; } diff --git a/src/AyonCppApi/AyonCppApi.h b/src/AyonCppApi/AyonCppApi.h index 2f7b347..81cf694 100644 --- a/src/AyonCppApi/AyonCppApi.h +++ b/src/AyonCppApi/AyonCppApi.h @@ -110,6 +110,21 @@ class AyonApi { */ std::unordered_map batchResolvePath(std::vector &uriPaths); + /** + * @brief Resolves a vector of paths in a SINGLE batched request over the persistent + * keep-alive client. + * + * Unlike batchResolvePath (which fans out parallel requests on fresh, non-keep-alive + * clients), this sends one POST with all URIs through m_ayonServer and parses the whole + * response array. One round-trip, deterministic, connection reused. Intended for the + * prewarm pass where frontiers are modest and determinism + keep-alive matter more than + * request-level parallelism. + * + * @param uriPaths The vector of URI paths to resolve. + * @return An unordered map of URI -> resolved path. + */ + std::unordered_map batchResolvePathSerial(const std::vector &uriPaths); + /** * @brief Takes an AYON path URI response (resolved ayon://path) and returns a pair of * asset identifier (ayon:// path) and the machine local file location. @@ -187,6 +202,10 @@ class AyonApi { // Core Dependencies std::unique_ptr m_ayonServer; std::shared_ptr m_log; + // Path to the CA cert bundle actually in use, recorded so a client + // can be rebuilt with the same configuration after a verification + // failure, without re-running cert path discovery. + std::string m_caCertPath; // Configuration from Constructor const std::string m_authKey; @@ -214,7 +233,13 @@ class AyonApi { uint16_t m_regroupSizeForAsyncRequests = 200; uint16_t m_maxGroupSizeForAsyncRequests = 300; uint16_t m_minVecSizeForGroupSplitAsyncRequests = 50; - + // Max uris per POST in batchResolvePathSerial. The server resolves a batch in one + // sequential transaction, so an unbounded request holds a DB connection for the whole + // frontier (risking its connection-pool 503 guard) and grows the body/response without + // limit. Splitting into capped chunks releases the connection between chunks. Matches + // m_regroupSizeForAsyncRequests so the serial and async paths cap requests alike. + uint16_t m_maxSerialBatchSize = 200; + // Retry and Timeout Configuration uint8_t m_maxCallRetries = 8; uint16_t m_retryWait = 800;