diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 24a76628fec..e7b87553817 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,10 @@ on: required: true default: true type: boolean + secrets: + GITBOOK_TOKEN: + description: 'Token used to publish versioned documentation' + required: true jobs: get_dry_release_versions: @@ -196,3 +200,27 @@ jobs: run: | git checkout -B stable origin/${GITHUB_REF#refs/heads/} git push origin stable --force + + publish_versioned_docs: + name: Publish versioned documentation + if: github.event.inputs.dry_run == 'false' + runs-on: ubuntu-latest + needs: [get_dry_release_versions, release] + permissions: + contents: write + env: + GITHUB_TOKEN: ${{ github.event.inputs.token }} + GITBOOK_TOKEN: ${{ secrets.GITBOOK_TOKEN }} + GITBOOK_ORG_ID: ${{ vars.GITBOOK_ORG_ID }} + GITBOOK_SITE_ID: ${{ vars.GITBOOK_SITE_ID }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Publish released documentation + run: | + python infra/scripts/release/publish_versioned_docs.py \ + --version "${{ needs.get_dry_release_versions.outputs.next_version }}" diff --git a/docs/project/release-process.md b/docs/project/release-process.md index 2cddca508cf..fd13c024474 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -110,9 +110,19 @@ We verify the building the wheels and Docker images in **your fork** of Feast, n - [Maven repo (feast-datatypes, feast-serving-client)](https://mvnrepository.com/artifact/dev.feast) ### 4. (for minor releases) Post-release steps -#### 4a: Creating a new branch -Create a new branch based on master (i.e. v0.22-branch) and push to the main Feast repo. This will be where -cherry-picks go for future patch releases and where documentation will point. +#### 4a: Versioned branch and documentation publication + +The `publish_versioned_docs` release job creates the `v.-branch` maintenance branch +at the released tag, imports that branch into a versioned GitBook space, and makes the new space +the public default. It verifies the final GitBook state before succeeding. + +The repository must define `GITBOOK_ORG_ID` and `GITBOOK_SITE_ID` variables and a scoped +`GITBOOK_TOKEN` secret with permission to create and update spaces, update the site, and publish +it. The release token must be able to create the maintenance branch. + +The job is safe to rerun. It reuses matching branches and spaces, but it will not move an existing +maintenance branch that points somewhere other than the released tag. It also leaves the previous +documentation version as the public default until the new branch has imported successfully. #### 4b: Adding a high level summary in the GitHub release notes By default, Semantic Release will pull in messages from commits (features vs fixes, etc). But this is hard to digest, @@ -120,7 +130,9 @@ so it helps to have a high level overview. See https://github.com/feast-dev/feas #### 4c: Update documentation -In the Feast Gitbook: +If the automated documentation job fails, use its error as the recovery point and complete the +remaining steps in the Feast GitBook: + 1. Create a new space within the Feast collection 2. Go to the overflow menu on the top -> Synchronize with Git 1. Specify GitHub as the provider diff --git a/infra/scripts/release/publish_versioned_docs.py b/infra/scripts/release/publish_versioned_docs.py new file mode 100644 index 00000000000..cf982909645 --- /dev/null +++ b/infra/scripts/release/publish_versioned_docs.py @@ -0,0 +1,342 @@ +"""Publish a released Feast minor version as the default GitBook documentation. + +The release workflow supplies the released semantic version and credentials through +environment variables. The operation is idempotent: matching branches and GitBook +spaces are reused, while conflicting release branches fail without being moved. +""" + +import argparse +import json +import os +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, NamedTuple + + +GITHUB_API_URL = "https://api.github.com" +GITBOOK_API_URL = "https://api.gitbook.com/v1" +GITHUB_API_VERSION = "2022-11-28" + + +class ApiError(RuntimeError): + def __init__(self, status: int, message: str): + super().__init__(message) + self.status = status + + +class VersionInfo(NamedTuple): + version: str + tag: str + branch: str + + @classmethod + def parse(cls, value: str) -> "VersionInfo": + match = re.fullmatch(r"v?(\d+)\.(\d+)\.(\d+)", value.strip()) + if match is None: + raise ValueError(f"expected a stable semantic version, got {value!r}") + major, minor, patch = match.groups() + version = f"{major}.{minor}.{patch}" + return cls(version, f"v{version}", f"v{major}.{minor}-branch") + + +class JsonApi: + def __init__( + self, + base_url: str, + token: str, + extra_headers: dict[str, str] | None = None, + ) -> None: + self.base_url = base_url.rstrip("/") + self.token = token + self.extra_headers = extra_headers or {} + + def request( + self, + method: str, + path: str, + payload: dict[str, Any] | None = None, + expected: tuple[int, ...] = (200,), + ) -> Any: + body = json.dumps(payload).encode() if payload is not None else None + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self.token}", + "User-Agent": "feast-release-docs", + **self.extra_headers, + } + if body is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + f"{self.base_url}{path}", data=body, headers=headers, method=method + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + status = response.status + response_body = response.read() + except urllib.error.HTTPError as error: + status = error.code + response_body = error.read() + except urllib.error.URLError as error: + raise RuntimeError( + f"request to {self.base_url} failed: {error.reason}" + ) from error + + if status not in expected: + detail = response_body.decode(errors="replace").strip() + raise ApiError(status, f"{method} {path} returned HTTP {status}: {detail}") + if not response_body: + return None + return json.loads(response_body) + + +class GitHubPublisher: + def __init__(self, api: JsonApi, repository: str) -> None: + self.api = api + self.repository = repository + self.repository_path = urllib.parse.quote(repository, safe="/") + + def _commit_sha(self, tag: str) -> str: + encoded_tag = urllib.parse.quote(tag, safe="") + commit = self.api.request( + "GET", f"/repos/{self.repository_path}/commits/{encoded_tag}" + ) + return commit["sha"] + + def _branch_sha(self, branch: str) -> str | None: + encoded_branch = urllib.parse.quote(branch, safe="") + try: + result = self.api.request( + "GET", f"/repos/{self.repository_path}/branches/{encoded_branch}" + ) + except ApiError as error: + if error.status == 404: + return None + raise + return result["commit"]["sha"] + + def ensure_release_branch(self, tag: str, branch: str) -> str: + tag_sha = self._commit_sha(tag) + branch_sha = self._branch_sha(branch) + if branch_sha is None: + self.api.request( + "POST", + f"/repos/{self.repository_path}/git/refs", + {"ref": f"refs/heads/{branch}", "sha": tag_sha}, + expected=(201,), + ) + print(f"Created {branch} at {tag_sha}") + return tag_sha + if branch_sha != tag_sha: + raise RuntimeError( + f"refusing to move existing {branch}: {branch_sha} != released {tag_sha}" + ) + print(f"Verified existing {branch} at {tag_sha}") + return tag_sha + + +class GitBookPublisher: + def __init__( + self, + api: JsonApi, + organization_id: str, + site_id: str, + repository: str, + sync_timeout: int = 180, + sync_poll_interval: int = 5, + ) -> None: + self.api = api + self.organization_id = urllib.parse.quote(organization_id, safe="") + self.site_id = urllib.parse.quote(site_id, safe="") + self.repository = repository + self.sync_timeout = sync_timeout + self.sync_poll_interval = sync_poll_interval + + @property + def _site_path(self) -> str: + return f"/orgs/{self.organization_id}/sites/{self.site_id}" + + def _list_site_spaces(self, default: bool | None = None) -> list[dict[str, Any]]: + query = {"limit": "1000"} + if default is not None: + query["default"] = str(default).lower() + result = self.api.request( + "GET", f"{self._site_path}/site-spaces?{urllib.parse.urlencode(query)}" + ) + return result["items"] + + @staticmethod + def _space_title(site_space: dict[str, Any]) -> str | None: + return site_space.get("space", {}).get("title") or site_space.get("title") + + def _find_site_space(self, title: str) -> dict[str, Any] | None: + return next( + ( + site_space + for site_space in self._list_site_spaces() + if self._space_title(site_space) == title + ), + None, + ) + + def _duplicate_default_space(self, title: str) -> dict[str, Any]: + defaults = self._list_site_spaces(default=True) + if len(defaults) != 1: + raise RuntimeError( + f"expected one default GitBook site space, found {len(defaults)}" + ) + source_id = urllib.parse.quote(defaults[0]["id"], safe="") + site_space = self.api.request( + "POST", + f"{self._site_path}/site-spaces/{source_id}/duplicate", + {"draft": True}, + expected=(201,), + ) + space_id = urllib.parse.quote(site_space["space"]["id"], safe="") + self.api.request("PATCH", f"/spaces/{space_id}", {"title": title}) + site_space["space"]["title"] = title + return site_space + + def _import_branch(self, space_id: str, branch: str) -> None: + encoded_space_id = urllib.parse.quote(space_id, safe="") + self.api.request( + "POST", + f"/spaces/{encoded_space_id}/git/import", + { + "url": f"https://github.com/{self.repository}", + "ref": f"refs/heads/{branch}", + "force": True, + }, + expected=(204,), + ) + + def _wait_for_import(self, space_id: str, branch: str) -> None: + encoded_space_id = urllib.parse.quote(space_id, safe="") + expected_suffix = f"/tree/{branch}" + deadline = time.monotonic() + self.sync_timeout + while True: + info = self.api.request("GET", f"/spaces/{encoded_space_id}/git/info") + operation = info.get("operation") or {} + state = operation.get("state") + branch_matches = info.get("url", "").rstrip("/").endswith(expected_suffix) + if branch_matches and state == "success": + return + if branch_matches and state in {"failure", "timeout"}: + detail = operation.get("error") or state + raise RuntimeError(f"GitBook import for {branch} failed: {detail}") + if time.monotonic() >= deadline: + raise RuntimeError( + f"timed out waiting for GitBook to import {branch} after " + f"{self.sync_timeout}s" + ) + time.sleep(self.sync_poll_interval) + + def _wait_for_default(self, site_space_id: str, branch: str) -> None: + deadline = time.monotonic() + self.sync_timeout + while True: + defaults = self._list_site_spaces(default=True) + if ( + len(defaults) == 1 + and defaults[0]["id"] == site_space_id + and self._space_title(defaults[0]) == branch + and not defaults[0].get("draft", False) + ): + return + if time.monotonic() >= deadline: + raise RuntimeError( + f"GitBook did not publish {branch} as the default space after " + f"{self.sync_timeout}s" + ) + time.sleep(self.sync_poll_interval) + + def publish_version(self, branch: str) -> None: + site_space = self._find_site_space(branch) + if site_space is None: + site_space = self._duplicate_default_space(branch) + print(f"Created draft GitBook space for {branch}") + else: + print(f"Reusing GitBook space for {branch}") + + site_space_id = site_space["id"] + space_id = site_space["space"]["id"] + self._import_branch(space_id, branch) + self._wait_for_import(space_id, branch) + + encoded_site_space_id = urllib.parse.quote(site_space_id, safe="") + self.api.request( + "PATCH", + f"{self._site_path}/site-spaces/{encoded_site_space_id}", + {"path": branch, "draft": False}, + ) + self.api.request("PATCH", self._site_path, {"defaultSiteSpace": site_space_id}) + self.api.request("POST", f"{self._site_path}/publish") + self._wait_for_default(site_space_id, branch) + print(f"Published {branch} as the default GitBook documentation") + + +def publish_release( + version: VersionInfo, + github: GitHubPublisher, + gitbook: GitBookPublisher, +) -> None: + github.ensure_release_branch(version.tag, version.branch) + gitbook.publish_version(version.branch) + + +def _required_env(name: str) -> str: + value = os.environ.get(name) + if not value: + raise SystemExit(f"error: {name} is required") + return value + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--version", required=True, help="released version, such as 0.66.0" + ) + parser.add_argument( + "--repository", + default=os.environ.get("GITHUB_REPOSITORY", "feast-dev/feast"), + help="GitHub owner/repository", + ) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--sync-timeout", type=int, default=180) + args = parser.parse_args() + + try: + version = VersionInfo.parse(args.version) + except ValueError as error: + raise SystemExit(f"error: {error}") from error + + if args.dry_run: + print( + f"Would publish {version.tag} from {version.branch} as the default " + "GitBook documentation" + ) + return + + github_api = JsonApi( + GITHUB_API_URL, + _required_env("GITHUB_TOKEN"), + { + "X-GitHub-Api-Version": GITHUB_API_VERSION, + "Accept": "application/vnd.github+json", + }, + ) + gitbook_api = JsonApi(GITBOOK_API_URL, _required_env("GITBOOK_TOKEN")) + github = GitHubPublisher(github_api, args.repository) + gitbook = GitBookPublisher( + gitbook_api, + _required_env("GITBOOK_ORG_ID"), + _required_env("GITBOOK_SITE_ID"), + args.repository, + sync_timeout=args.sync_timeout, + ) + publish_release(version, github, gitbook) + + +if __name__ == "__main__": + main() diff --git a/infra/scripts/release/test_publish_versioned_docs.py b/infra/scripts/release/test_publish_versioned_docs.py new file mode 100644 index 00000000000..536ecd7153e --- /dev/null +++ b/infra/scripts/release/test_publish_versioned_docs.py @@ -0,0 +1,246 @@ +import importlib.util +import sys +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("publish_versioned_docs.py") +SPEC = importlib.util.spec_from_file_location("publish_versioned_docs", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +publisher = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = publisher +SPEC.loader.exec_module(publisher) + + +class FakeApi: + def __init__(self, responses): + self.responses = list(responses) + self.calls = [] + + def request(self, method, path, payload=None, expected=(200,)): + self.calls.append((method, path, payload, expected)) + response = self.responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + +def site_space(identifier, title, *, default=False, draft=False): + return { + "id": identifier, + "title": title, + "default": default, + "draft": draft, + "space": {"id": f"space-{identifier}", "title": title}, + } + + +def successful_import(branch): + return { + "url": f"https://github.com/feast-dev/feast/tree/{branch}", + "operation": {"state": "success", "direction": "import"}, + } + + +class VersionInfoTests(unittest.TestCase): + def test_parses_release_version(self): + self.assertEqual( + publisher.VersionInfo.parse("v0.66.0"), + ("0.66.0", "v0.66.0", "v0.66-branch"), + ) + + def test_rejects_prerelease_version(self): + with self.assertRaisesRegex(ValueError, "stable semantic version"): + publisher.VersionInfo.parse("0.66.0-rc1") + + +class GitHubPublisherTests(unittest.TestCase): + def test_creates_missing_release_branch_at_tag(self): + api = FakeApi( + [ + {"sha": "release-sha"}, + publisher.ApiError(404, "missing"), + {"ref": "refs/heads/v0.66-branch"}, + ] + ) + github = publisher.GitHubPublisher(api, "feast-dev/feast") + + self.assertEqual( + github.ensure_release_branch("v0.66.0", "v0.66-branch"), + "release-sha", + ) + self.assertEqual( + api.calls[-1], + ( + "POST", + "/repos/feast-dev/feast/git/refs", + {"ref": "refs/heads/v0.66-branch", "sha": "release-sha"}, + (201,), + ), + ) + + def test_reuses_matching_release_branch(self): + api = FakeApi([{"sha": "release-sha"}, {"commit": {"sha": "release-sha"}}]) + github = publisher.GitHubPublisher(api, "feast-dev/feast") + + github.ensure_release_branch("v0.66.0", "v0.66-branch") + + self.assertEqual(len(api.calls), 2) + + def test_refuses_to_move_conflicting_release_branch(self): + api = FakeApi([{"sha": "release-sha"}, {"commit": {"sha": "different-sha"}}]) + github = publisher.GitHubPublisher(api, "feast-dev/feast") + + with self.assertRaisesRegex(RuntimeError, "refusing to move"): + github.ensure_release_branch("v0.66.0", "v0.66-branch") + + self.assertEqual(len(api.calls), 2) + + +class GitBookPublisherTests(unittest.TestCase): + def _publisher(self, responses): + api = FakeApi(responses) + gitbook = publisher.GitBookPublisher( + api, + "org", + "site", + "feast-dev/feast", + sync_timeout=0, + sync_poll_interval=0, + ) + return gitbook, api + + def test_creates_imports_and_publishes_new_version(self): + old_default = site_space("old", "v0.64-branch", default=True) + new_space = site_space("new", "v0.64-branch", draft=True) + new_default = site_space("new", "v0.66-branch", default=True) + gitbook, api = self._publisher( + [ + {"items": [old_default]}, + {"items": [old_default]}, + new_space, + {}, + None, + successful_import("v0.66-branch"), + {}, + {}, + {}, + {"items": [new_default]}, + ] + ) + + gitbook.publish_version("v0.66-branch") + + self.assertIn( + ( + "POST", + "/orgs/org/sites/site/site-spaces/old/duplicate", + {"draft": True}, + (201,), + ), + api.calls, + ) + self.assertIn( + ( + "PATCH", + "/orgs/org/sites/site", + {"defaultSiteSpace": "new"}, + (200,), + ), + api.calls, + ) + + def test_reuses_existing_version_space(self): + target = site_space("new", "v0.66-branch") + new_default = site_space("new", "v0.66-branch", default=True) + gitbook, api = self._publisher( + [ + {"items": [target]}, + None, + successful_import("v0.66-branch"), + {}, + {}, + {}, + {"items": [new_default]}, + ] + ) + + gitbook.publish_version("v0.66-branch") + + self.assertFalse(any(call[1].endswith("/duplicate") for call in api.calls)) + + def test_failed_import_does_not_change_public_default(self): + target = site_space("new", "v0.66-branch", draft=True) + gitbook, api = self._publisher( + [ + {"items": [target]}, + None, + { + "url": "https://github.com/feast-dev/feast/tree/v0.66-branch", + "operation": { + "state": "failure", + "direction": "import", + "error": "invalid content", + }, + }, + ] + ) + + with self.assertRaisesRegex(RuntimeError, "invalid content"): + gitbook.publish_version("v0.66-branch") + + self.assertFalse(any(call[0] == "PATCH" for call in api.calls)) + + def test_fails_when_default_space_readback_does_not_match(self): + target = site_space("new", "v0.66-branch") + old_default = site_space("old", "v0.64-branch", default=True) + gitbook, _api = self._publisher( + [ + {"items": [target]}, + None, + successful_import("v0.66-branch"), + {}, + {}, + {}, + {"items": [old_default]}, + ] + ) + + with self.assertRaisesRegex(RuntimeError, "did not publish"): + gitbook.publish_version("v0.66-branch") + + def test_waits_for_default_space_readback_to_match(self): + target = site_space("new", "v0.66-branch") + old_default = site_space("old", "v0.64-branch", default=True) + new_default = site_space("new", "v0.66-branch", default=True) + api = FakeApi( + [ + {"items": [target]}, + None, + successful_import("v0.66-branch"), + {}, + {}, + {}, + {"items": [old_default]}, + {"items": [new_default]}, + ] + ) + gitbook = publisher.GitBookPublisher( + api, + "org", + "site", + "feast-dev/feast", + sync_timeout=1, + sync_poll_interval=0, + ) + + gitbook.publish_version("v0.66-branch") + + default_reads = [ + call for call in api.calls if call[0] == "GET" and "default=true" in call[1] + ] + self.assertEqual(len(default_reads), 2) + + +if __name__ == "__main__": + unittest.main()