From b75cfffe2fb345a8ed37097c0f9920dfcd0f24ea Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Tue, 8 Nov 2022 09:09:55 +0100 Subject: [PATCH 01/32] X-Smart-Branch-Parent: master From b1dda7728e246a70413940677a24b2292ddee328 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Wed, 26 Oct 2022 18:46:07 +0200 Subject: [PATCH 02/32] Add sensorversions.py script (WIP) --- .openshift-ci/sensorversions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .openshift-ci/sensorversions.py diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py new file mode 100644 index 0000000000000..8cb278a94f316 --- /dev/null +++ b/.openshift-ci/sensorversions.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +""" +Returns the latest patches of the last 4 major versions +""" + +import sys +import re + +def splitVersion(version): + digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) + return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) + +def imageExists(version): + x, y, z = splitVersion(version) + return (y > z) + +def getLast4SensorVersions(current_version): + x,y,z = splitVersion(current_version) + latestversions = [] + for y_test in range(y-3,y+1): + z_test = 0 + while imageExists("quay.io/stackrox-io/main:" + str(x) + "." + str(y_test) + "." + str(z_test+1)): + z_test += 1 + latestversions.append(str(x) + "." + str(y_test) + "." + str(z_test)) + return latestversions + +def main(argv): + latestversions = getLast4SensorVersions(argv[1]) + print(str(latestversions[0]) + " " + str(latestversions[1]) + " " + str(latestversions[2]) + " " + str(latestversions[3])) + +if (__name__ == "__main__"): + main(sys.argv) From e92a30e32a2026763846ccf233d205ccde93fa0c Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Wed, 26 Oct 2022 19:19:18 +0200 Subject: [PATCH 03/32] Add TODOs for remaining changes --- .openshift-ci/sensorversions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index 8cb278a94f316..ab11bd1d987b8 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -11,10 +11,12 @@ def splitVersion(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) +# TODO: figure out how to check if an image exists and check in this function def imageExists(version): x, y, z = splitVersion(version) return (y > z) +# TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done def getLast4SensorVersions(current_version): x,y,z = splitVersion(current_version) latestversions = [] From b870acf39387d890051c17932a8a5ff2c51a5a96 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Fri, 28 Oct 2022 11:06:13 +0200 Subject: [PATCH 04/32] Finish quay tag querying --- .openshift-ci/sensorversions.py | 70 +++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index ab11bd1d987b8..4bb48c33028b2 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -6,30 +6,74 @@ import sys import re +import requests +import json + +def isReleaseVersion(version): + return bool(re.search(r"\d+\.\d+\.\d+$", version)) + +def filterTags(tags): + filteredtags = [] + for t in tags: + name = t['name'] + if isReleaseVersion(name): + filteredtags.append(name) + return filteredtags + +def queryQuayForTags(): + tags = [] + page = 1 + pageNotEmpty = True + while pageNotEmpty: + print("Going through page " + str(page)) + apiresponse = requests.get("https://quay.io/api/v1/repository/stackrox-io/main/tag/?page=" + str(page) + "&limit=100") + rawtags = apiresponse.json()['tags'] + tags.extend(filterTags(rawtags)) + if not bool(rawtags): + pageNotEmpty = False + page+=1 + return sorted(set(tags)) + +def transformTagsToNumbers(tags): + numTags = [] + for t in tags: + numTags.append(splitVersion(t)) + return numTags def splitVersion(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) -# TODO: figure out how to check if an image exists and check in this function -def imageExists(version): - x, y, z = splitVersion(version) - return (y > z) - # TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done -def getLast4SensorVersions(current_version): +def getLastSensorVersionsFromQuay(current_version, num_versions): + tags = queryQuayForTags() + numericaltags = transformTagsToNumbers(tags) + print(numericaltags) + print(numericaltags[::-1]) + x,y,z = splitVersion(current_version) + latestversions = [] - for y_test in range(y-3,y+1): - z_test = 0 - while imageExists("quay.io/stackrox-io/main:" + str(x) + "." + str(y_test) + "." + str(z_test+1)): - z_test += 1 - latestversions.append(str(x) + "." + str(y_test) + "." + str(z_test)) + + ycurr = y + for tags in numericaltags[::-1]: + if tags[1] < y-num_versions+1: + break + if tags[1] < ycurr: + ycurr = tags[1] + if tags[1] == ycurr: + latestversions.append(str(tags[0]) + "." + str(tags[1]) + "." + str(tags[2])) + ycurr-=1 + return latestversions def main(argv): - latestversions = getLast4SensorVersions(argv[1]) - print(str(latestversions[0]) + " " + str(latestversions[1]) + " " + str(latestversions[2]) + " " + str(latestversions[3])) + latestversions = getLastSensorVersionsFromQuay(argv[1], 4) + printversions = "" + for version in latestversions: + printversions += str(version) + " " + print(printversions) + if (__name__ == "__main__"): main(sys.argv) From 347a12d1d27b13c40ec1bf00fdc66f29209e1ea2 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Fri, 28 Oct 2022 11:24:15 +0200 Subject: [PATCH 05/32] Add git tag API query --- .openshift-ci/sensorversions.py | 53 +++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index 4bb48c33028b2..ea7ca8367f725 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -10,9 +10,9 @@ import json def isReleaseVersion(version): - return bool(re.search(r"\d+\.\d+\.\d+$", version)) + return bool(re.search(r"^\D+\d+\.\d+\.\d+$", version)) -def filterTags(tags): +def filterQuayTags(tags): filteredtags = [] for t in tags: name = t['name'] @@ -28,17 +28,17 @@ def queryQuayForTags(): print("Going through page " + str(page)) apiresponse = requests.get("https://quay.io/api/v1/repository/stackrox-io/main/tag/?page=" + str(page) + "&limit=100") rawtags = apiresponse.json()['tags'] - tags.extend(filterTags(rawtags)) + tags.extend(filterQuayTags(rawtags)) if not bool(rawtags): pageNotEmpty = False page+=1 - return sorted(set(tags)) + return set(tags) def transformTagsToNumbers(tags): numTags = [] for t in tags: numTags.append(splitVersion(t)) - return numTags + return sorted(numTags) def splitVersion(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) @@ -48,13 +48,8 @@ def splitVersion(version): def getLastSensorVersionsFromQuay(current_version, num_versions): tags = queryQuayForTags() numericaltags = transformTagsToNumbers(tags) - print(numericaltags) - print(numericaltags[::-1]) - x,y,z = splitVersion(current_version) - latestversions = [] - ycurr = y for tags in numericaltags[::-1]: if tags[1] < y-num_versions+1: @@ -64,16 +59,50 @@ def getLastSensorVersionsFromQuay(current_version, num_versions): if tags[1] == ycurr: latestversions.append(str(tags[0]) + "." + str(tags[1]) + "." + str(tags[2])) ycurr-=1 + return latestversions + +def filterGitTags(): + apiresponse = requests.get("https://api.github.com/repos/stackrox/stackrox/git/refs/tags") + rawtags = apiresponse.json() + filteredtags = [] + for t in rawtags: + name = t['ref'] + if isReleaseVersion(name): + filteredtags.append(name) + return set(filteredtags) + +def jprint(obj): + # create a formatted string of the Python JSON object + text = json.dumps(obj, sort_keys=True, indent=4) + print(text) + + +# TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done +def getLastSensorVersionsFromGitTags(current_version, num_versions): + tags = filterGitTags() + numericaltags = transformTagsToNumbers(tags) + x,y,z = splitVersion(current_version) + latestversions = [] + ycurr = y + for tags in numericaltags[::-1]: + if tags[1] < y-num_versions+1: + break + if tags[1] < ycurr: + ycurr = tags[1] + if tags[1] == ycurr: + latestversions.append(str(tags[0]) + "." + str(tags[1]) + "." + str(tags[2])) + ycurr-=1 return latestversions + def main(argv): - latestversions = getLastSensorVersionsFromQuay(argv[1], 4) + latestversions = getLastSensorVersionsFromGitTags(argv[1], 4) + #latestversions = getLastSensorVersionsFromQuay(argv[1], 4) printversions = "" for version in latestversions: printversions += str(version) + " " print(printversions) - if (__name__ == "__main__"): main(sys.argv) From da3d579b5fe649082af8188fed81f879d9bf6465 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Fri, 28 Oct 2022 11:31:06 +0200 Subject: [PATCH 06/32] Make functions that don't need exposure private --- .openshift-ci/sensorversions.py | 60 ++++++++++++++------------------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index ea7ca8367f725..aa022410fc1d3 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -9,18 +9,18 @@ import requests import json -def isReleaseVersion(version): +def __isReleaseVersion(version): return bool(re.search(r"^\D+\d+\.\d+\.\d+$", version)) -def filterQuayTags(tags): +def __filterQuayTags(tags): filteredtags = [] for t in tags: name = t['name'] - if isReleaseVersion(name): + if __isReleaseVersion(name): filteredtags.append(name) return filteredtags -def queryQuayForTags(): +def __queryQuayForTags(): tags = [] page = 1 pageNotEmpty = True @@ -28,27 +28,37 @@ def queryQuayForTags(): print("Going through page " + str(page)) apiresponse = requests.get("https://quay.io/api/v1/repository/stackrox-io/main/tag/?page=" + str(page) + "&limit=100") rawtags = apiresponse.json()['tags'] - tags.extend(filterQuayTags(rawtags)) + tags.extend(__filterQuayTags(rawtags)) if not bool(rawtags): pageNotEmpty = False page+=1 return set(tags) -def transformTagsToNumbers(tags): +def __transformTagsToNumbers(tags): numTags = [] for t in tags: - numTags.append(splitVersion(t)) + numTags.append(__splitVersion(t)) return sorted(numTags) -def splitVersion(version): +def __splitVersion(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) +def __filterGitTags(): + apiresponse = requests.get("https://api.github.com/repos/stackrox/stackrox/git/refs/tags") + rawtags = apiresponse.json() + filteredtags = [] + for t in rawtags: + name = t['ref'] + if __isReleaseVersion(name): + filteredtags.append(name) + return set(filteredtags) + # TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done def getLastSensorVersionsFromQuay(current_version, num_versions): - tags = queryQuayForTags() - numericaltags = transformTagsToNumbers(tags) - x,y,z = splitVersion(current_version) + tags = __queryQuayForTags() + numericaltags = __transformTagsToNumbers(tags) + x,y,z = __splitVersion(current_version) latestversions = [] ycurr = y for tags in numericaltags[::-1]: @@ -61,28 +71,11 @@ def getLastSensorVersionsFromQuay(current_version, num_versions): ycurr-=1 return latestversions - -def filterGitTags(): - apiresponse = requests.get("https://api.github.com/repos/stackrox/stackrox/git/refs/tags") - rawtags = apiresponse.json() - filteredtags = [] - for t in rawtags: - name = t['ref'] - if isReleaseVersion(name): - filteredtags.append(name) - return set(filteredtags) - -def jprint(obj): - # create a formatted string of the Python JSON object - text = json.dumps(obj, sort_keys=True, indent=4) - print(text) - - # TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done def getLastSensorVersionsFromGitTags(current_version, num_versions): - tags = filterGitTags() - numericaltags = transformTagsToNumbers(tags) - x,y,z = splitVersion(current_version) + tags = __filterGitTags() + numericaltags = __transformTagsToNumbers(tags) + x,y,z = __splitVersion(current_version) latestversions = [] ycurr = y for tags in numericaltags[::-1]: @@ -95,10 +88,9 @@ def getLastSensorVersionsFromGitTags(current_version, num_versions): ycurr-=1 return latestversions - def main(argv): - latestversions = getLastSensorVersionsFromGitTags(argv[1], 4) - #latestversions = getLastSensorVersionsFromQuay(argv[1], 4) + #latestversions = getLastSensorVersionsFromGitTags(argv[1], 4) + latestversions = getLastSensorVersionsFromQuay(argv[1], 4) printversions = "" for version in latestversions: printversions += str(version) + " " From 1b7f9c894f11277969954cdcae2388a73fb6df9c Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Fri, 28 Oct 2022 12:21:16 +0200 Subject: [PATCH 07/32] Separate tag syntax filtering between git and quay tags --- .openshift-ci/sensorversions.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index aa022410fc1d3..12100b2f87440 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -9,14 +9,17 @@ import requests import json -def __isReleaseVersion(version): +def __isReleaseTagGit(version): return bool(re.search(r"^\D+\d+\.\d+\.\d+$", version)) +def __isReleaseTagQuay(version): + return bool(re.search(r"\d+\.\d+\.\d+$", version)) + def __filterQuayTags(tags): filteredtags = [] for t in tags: name = t['name'] - if __isReleaseVersion(name): + if __isReleaseTagQuay(name): filteredtags.append(name) return filteredtags @@ -50,11 +53,13 @@ def __filterGitTags(): filteredtags = [] for t in rawtags: name = t['ref'] - if __isReleaseVersion(name): + if __isReleaseTagGit(name): filteredtags.append(name) return set(filteredtags) # TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done +# getLastSensorVersionsFromQuay gets the latest patches of the last num_versions major versions +# querying quay API is slow, prefer using git tag API def getLastSensorVersionsFromQuay(current_version, num_versions): tags = __queryQuayForTags() numericaltags = __transformTagsToNumbers(tags) @@ -72,6 +77,8 @@ def getLastSensorVersionsFromQuay(current_version, num_versions): return latestversions # TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done +# getLastSensorVersionsFromGitTags gets the latest patches of the last num_versions major versions +# much faster than querying quay def getLastSensorVersionsFromGitTags(current_version, num_versions): tags = __filterGitTags() numericaltags = __transformTagsToNumbers(tags) @@ -89,8 +96,8 @@ def getLastSensorVersionsFromGitTags(current_version, num_versions): return latestversions def main(argv): - #latestversions = getLastSensorVersionsFromGitTags(argv[1], 4) - latestversions = getLastSensorVersionsFromQuay(argv[1], 4) + latestversions = getLastSensorVersionsFromGitTags(argv[1], 4) + #latestversions = getLastSensorVersionsFromQuay(argv[1], 4) printversions = "" for version in latestversions: printversions += str(version) + " " From ac0c621c4d74b36cda6fefa6e8208041468affd8 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Fri, 28 Oct 2022 13:08:14 +0200 Subject: [PATCH 08/32] Add Git CLI version fetching --- .openshift-ci/sensorversions.py | 73 +++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index 12100b2f87440..a710221bf7538 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -8,9 +8,13 @@ import re import requests import json +import subprocess -def __isReleaseTagGit(version): - return bool(re.search(r"^\D+\d+\.\d+\.\d+$", version)) +def __isReleaseTagGitAPI(version): + return bool(re.search(r"^refs/tags/\d+\.\d+\.\d+$", version)) + +def __isReleaseTagGitCLI(version): + return bool(re.search(r"^\d+\.\d+\.\d+$", version)) def __isReleaseTagQuay(version): return bool(re.search(r"\d+\.\d+\.\d+$", version)) @@ -47,23 +51,28 @@ def __splitVersion(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) -def __filterGitTags(): - apiresponse = requests.get("https://api.github.com/repos/stackrox/stackrox/git/refs/tags") - rawtags = apiresponse.json() +def __filterGitTags(rawtags): filteredtags = [] for t in rawtags: name = t['ref'] - if __isReleaseTagGit(name): + if __isReleaseTagGitAPI(name): filteredtags.append(name) return set(filteredtags) -# TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done -# getLastSensorVersionsFromQuay gets the latest patches of the last num_versions major versions -# querying quay API is slow, prefer using git tag API -def getLastSensorVersionsFromQuay(current_version, num_versions): - tags = __queryQuayForTags() +def __cliOutputToTags(stdoutput): + separated = stdoutput.decode(encoding="utf-8").splitlines() + return separated + +def __filterGitCLITags(rawtags): + filteredtags = [] + for t in rawtags: + if __isReleaseTagGitCLI(t): + filteredtags.append(t) + return set(filteredtags) + +def __getLatestTags(current_version, tags, num_versions): numericaltags = __transformTagsToNumbers(tags) - x,y,z = __splitVersion(current_version) + _,y,_ = __splitVersion(current_version) latestversions = [] ycurr = y for tags in numericaltags[::-1]: @@ -77,27 +86,31 @@ def getLastSensorVersionsFromQuay(current_version, num_versions): return latestversions # TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done -# getLastSensorVersionsFromGitTags gets the latest patches of the last num_versions major versions +# getLastSensorVersionsFromQuay gets the latest patches of the last num_versions major versions +# querying quay API is slow, prefer using git tag API +def getLastSensorVersionsFromQuayAPI(current_version, num_versions): + tags = __queryQuayForTags() + return __getLatestTags(current_version, tags, num_versions) + +# TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done +# getLastSensorVersionsFromGitTagsAPI gets the latest patches of the last num_versions major versions via Git API # much faster than querying quay -def getLastSensorVersionsFromGitTags(current_version, num_versions): - tags = __filterGitTags() - numericaltags = __transformTagsToNumbers(tags) - x,y,z = __splitVersion(current_version) - latestversions = [] - ycurr = y - for tags in numericaltags[::-1]: - if tags[1] < y-num_versions+1: - break - if tags[1] < ycurr: - ycurr = tags[1] - if tags[1] == ycurr: - latestversions.append(str(tags[0]) + "." + str(tags[1]) + "." + str(tags[2])) - ycurr-=1 - return latestversions +def getLastSensorVersionsFromGitTagsAPI(current_version, num_versions): + apiresponse = requests.get("https://api.github.com/repos/stackrox/stackrox/git/refs/tags") + tags = __filterGitTags(apiresponse.json()) + return __getLatestTags(current_version, tags, num_versions) + +# getLastSensorVersionsFromGitTagsCLI gets the latest patches of the last num_versions major versions via Git CLI +# preferably use this to avoid API calls if possible +def getLastSensorVersionsFromGitTagsCLI(current_version, num_versions): + rawtags = __cliOutputToTags(subprocess.check_output(["git", "tag", "--list"])) + tags = __filterGitCLITags(rawtags) + return __getLatestTags(current_version, tags, num_versions) def main(argv): - latestversions = getLastSensorVersionsFromGitTags(argv[1], 4) - #latestversions = getLastSensorVersionsFromQuay(argv[1], 4) + latestversions = getLastSensorVersionsFromGitTagsCLI(argv[1], 4) + #latestversions = getLastSensorVersionsFromGitTagsAPI(argv[1], 4) + #latestversions = getLastSensorVersionsFromQuayAPI(argv[1], 4) printversions = "" for version in latestversions: printversions += str(version) + " " From bba20f1ed363fbdd713872e5a65652b1e1d43735 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Fri, 28 Oct 2022 13:09:37 +0200 Subject: [PATCH 09/32] Refactor --- .openshift-ci/sensorversions.py | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index a710221bf7538..05226ed5ed7cf 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -10,15 +10,15 @@ import json import subprocess +def __isReleaseTagQuay(version): + return bool(re.search(r"\d+\.\d+\.\d+$", version)) + def __isReleaseTagGitAPI(version): return bool(re.search(r"^refs/tags/\d+\.\d+\.\d+$", version)) def __isReleaseTagGitCLI(version): return bool(re.search(r"^\d+\.\d+\.\d+$", version)) -def __isReleaseTagQuay(version): - return bool(re.search(r"\d+\.\d+\.\d+$", version)) - def __filterQuayTags(tags): filteredtags = [] for t in tags: @@ -27,6 +27,21 @@ def __filterQuayTags(tags): filteredtags.append(name) return filteredtags +def __filterGitTags(rawtags): + filteredtags = [] + for t in rawtags: + name = t['ref'] + if __isReleaseTagGitAPI(name): + filteredtags.append(name) + return set(filteredtags) + +def __filterGitCLITags(rawtags): + filteredtags = [] + for t in rawtags: + if __isReleaseTagGitCLI(t): + filteredtags.append(t) + return set(filteredtags) + def __queryQuayForTags(): tags = [] page = 1 @@ -41,6 +56,10 @@ def __queryQuayForTags(): page+=1 return set(tags) +def __cliOutputToTags(stdoutput): + separated = stdoutput.decode(encoding="utf-8").splitlines() + return separated + def __transformTagsToNumbers(tags): numTags = [] for t in tags: @@ -51,25 +70,6 @@ def __splitVersion(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) -def __filterGitTags(rawtags): - filteredtags = [] - for t in rawtags: - name = t['ref'] - if __isReleaseTagGitAPI(name): - filteredtags.append(name) - return set(filteredtags) - -def __cliOutputToTags(stdoutput): - separated = stdoutput.decode(encoding="utf-8").splitlines() - return separated - -def __filterGitCLITags(rawtags): - filteredtags = [] - for t in rawtags: - if __isReleaseTagGitCLI(t): - filteredtags.append(t) - return set(filteredtags) - def __getLatestTags(current_version, tags, num_versions): numericaltags = __transformTagsToNumbers(tags) _,y,_ = __splitVersion(current_version) From ab0074983f9e6f29126843923ae1829b8b4f7123 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Fri, 4 Nov 2022 10:51:44 +0100 Subject: [PATCH 10/32] Remove API methods for getting tags, remove underscores in functions --- .openshift-ci/sensorversions.py | 81 ++++++--------------------------- 1 file changed, 13 insertions(+), 68 deletions(-) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index 05226ed5ed7cf..ab2b8a687a0b3 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -6,73 +6,35 @@ import sys import re -import requests -import json import subprocess -def __isReleaseTagQuay(version): - return bool(re.search(r"\d+\.\d+\.\d+$", version)) - -def __isReleaseTagGitAPI(version): - return bool(re.search(r"^refs/tags/\d+\.\d+\.\d+$", version)) - -def __isReleaseTagGitCLI(version): +def isReleaseTagGitCLI(version): return bool(re.search(r"^\d+\.\d+\.\d+$", version)) -def __filterQuayTags(tags): - filteredtags = [] - for t in tags: - name = t['name'] - if __isReleaseTagQuay(name): - filteredtags.append(name) - return filteredtags - -def __filterGitTags(rawtags): - filteredtags = [] - for t in rawtags: - name = t['ref'] - if __isReleaseTagGitAPI(name): - filteredtags.append(name) - return set(filteredtags) - -def __filterGitCLITags(rawtags): +def filterGitCLITags(rawtags): filteredtags = [] for t in rawtags: - if __isReleaseTagGitCLI(t): + if isReleaseTagGitCLI(t): filteredtags.append(t) return set(filteredtags) -def __queryQuayForTags(): - tags = [] - page = 1 - pageNotEmpty = True - while pageNotEmpty: - print("Going through page " + str(page)) - apiresponse = requests.get("https://quay.io/api/v1/repository/stackrox-io/main/tag/?page=" + str(page) + "&limit=100") - rawtags = apiresponse.json()['tags'] - tags.extend(__filterQuayTags(rawtags)) - if not bool(rawtags): - pageNotEmpty = False - page+=1 - return set(tags) - -def __cliOutputToTags(stdoutput): +def cliOutputToTags(stdoutput): separated = stdoutput.decode(encoding="utf-8").splitlines() return separated -def __transformTagsToNumbers(tags): +def transformTagsToNumbers(tags): numTags = [] for t in tags: - numTags.append(__splitVersion(t)) + numTags.append(splitVersion(t)) return sorted(numTags) -def __splitVersion(version): +def splitVersion(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) -def __getLatestTags(current_version, tags, num_versions): - numericaltags = __transformTagsToNumbers(tags) - _,y,_ = __splitVersion(current_version) +def getLatestTags(current_version, tags, num_versions): + numericaltags = transformTagsToNumbers(tags) + _,y,_ = splitVersion(current_version) latestversions = [] ycurr = y for tags in numericaltags[::-1]: @@ -85,32 +47,15 @@ def __getLatestTags(current_version, tags, num_versions): ycurr-=1 return latestversions -# TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done -# getLastSensorVersionsFromQuay gets the latest patches of the last num_versions major versions -# querying quay API is slow, prefer using git tag API -def getLastSensorVersionsFromQuayAPI(current_version, num_versions): - tags = __queryQuayForTags() - return __getLatestTags(current_version, tags, num_versions) - -# TODO: grab current_image from os.environ["MAIN_IMAGE_TAG"] after manual testing is done -# getLastSensorVersionsFromGitTagsAPI gets the latest patches of the last num_versions major versions via Git API -# much faster than querying quay -def getLastSensorVersionsFromGitTagsAPI(current_version, num_versions): - apiresponse = requests.get("https://api.github.com/repos/stackrox/stackrox/git/refs/tags") - tags = __filterGitTags(apiresponse.json()) - return __getLatestTags(current_version, tags, num_versions) - # getLastSensorVersionsFromGitTagsCLI gets the latest patches of the last num_versions major versions via Git CLI # preferably use this to avoid API calls if possible def getLastSensorVersionsFromGitTagsCLI(current_version, num_versions): - rawtags = __cliOutputToTags(subprocess.check_output(["git", "tag", "--list"])) - tags = __filterGitCLITags(rawtags) - return __getLatestTags(current_version, tags, num_versions) + rawtags = cliOutputToTags(subprocess.check_output(["git", "tag", "--list"])) + tags = filterGitCLITags(rawtags) + return getLatestTags(current_version, tags, num_versions) def main(argv): latestversions = getLastSensorVersionsFromGitTagsCLI(argv[1], 4) - #latestversions = getLastSensorVersionsFromGitTagsAPI(argv[1], 4) - #latestversions = getLastSensorVersionsFromQuayAPI(argv[1], 4) printversions = "" for version in latestversions: printversions += str(version) + " " From b9716e65f8af5d65c25667d404e570520d86e99e Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Fri, 4 Nov 2022 10:52:54 +0100 Subject: [PATCH 11/32] Remove main function --- .openshift-ci/sensorversions.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensorversions.py index ab2b8a687a0b3..8f19301a5f29d 100644 --- a/.openshift-ci/sensorversions.py +++ b/.openshift-ci/sensorversions.py @@ -48,18 +48,7 @@ def getLatestTags(current_version, tags, num_versions): return latestversions # getLastSensorVersionsFromGitTagsCLI gets the latest patches of the last num_versions major versions via Git CLI -# preferably use this to avoid API calls if possible def getLastSensorVersionsFromGitTagsCLI(current_version, num_versions): rawtags = cliOutputToTags(subprocess.check_output(["git", "tag", "--list"])) tags = filterGitCLITags(rawtags) return getLatestTags(current_version, tags, num_versions) - -def main(argv): - latestversions = getLastSensorVersionsFromGitTagsCLI(argv[1], 4) - printversions = "" - for version in latestversions: - printversions += str(version) + " " - print(printversions) - -if (__name__ == "__main__"): - main(sys.argv) From 0a223567219b349000a8d5531e810ec01bf40245 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Tue, 8 Nov 2022 09:15:16 +0100 Subject: [PATCH 12/32] Rename sensorversions.py to sensor_versions.py to follow python naming conventions --- .openshift-ci/{sensorversions.py => sensor_versions.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .openshift-ci/{sensorversions.py => sensor_versions.py} (100%) diff --git a/.openshift-ci/sensorversions.py b/.openshift-ci/sensor_versions.py similarity index 100% rename from .openshift-ci/sensorversions.py rename to .openshift-ci/sensor_versions.py From 6ddd3564dc82aaa7d57c926025f20c3f41222095 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Tue, 8 Nov 2022 09:15:37 +0100 Subject: [PATCH 13/32] Use sensor_versions.py to grab the latest sensor versions for compatibility testing --- scripts/ci/jobs/gke_version_compatibility_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/jobs/gke_version_compatibility_tests.py b/scripts/ci/jobs/gke_version_compatibility_tests.py index 523b739496406..5899a0368679a 100755 --- a/scripts/ci/jobs/gke_version_compatibility_tests.py +++ b/scripts/ci/jobs/gke_version_compatibility_tests.py @@ -4,6 +4,7 @@ Run version compatibility tests """ import os +from sensor_versions import getLastSensorVersionsFromGitTagsCLI from compatibility_test import make_compatibility_test_runner from clusters import GKECluster @@ -13,7 +14,7 @@ # don't use postgres os.environ["ROX_POSTGRES_DATASTORE"] = "false" -versions=["3.71.0", "3.70.0", "3.69.0"] +versions=getLastSensorVersionsFromGitTagsCLI(os.environ["MAIN_IMAGE_TAG"], 4) gkecluster=GKECluster("qa-e2e-test") From 9936fd319d3bfde2866ecedbd8e379ef18822f44 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Tue, 8 Nov 2022 11:25:17 +0100 Subject: [PATCH 14/32] make tag within sensor_versions.py script to get MAIN_IMAGE_TAG --- .openshift-ci/sensor_versions.py | 11 +++++++---- scripts/ci/jobs/gke_version_compatibility_tests.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.openshift-ci/sensor_versions.py b/.openshift-ci/sensor_versions.py index 8f19301a5f29d..410b9c5a2915a 100644 --- a/.openshift-ci/sensor_versions.py +++ b/.openshift-ci/sensor_versions.py @@ -32,9 +32,9 @@ def splitVersion(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) -def getLatestTags(current_version, tags, num_versions): +def getLatestTags(tags, num_versions): numericaltags = transformTagsToNumbers(tags) - _,y,_ = splitVersion(current_version) + _,y,_ = splitVersion(makeImageTag()) latestversions = [] ycurr = y for tags in numericaltags[::-1]: @@ -47,8 +47,11 @@ def getLatestTags(current_version, tags, num_versions): ycurr-=1 return latestversions +def makeImageTag(): + return subprocess.check_output(["make", "tag"]).decode(encoding="utf-8") + # getLastSensorVersionsFromGitTagsCLI gets the latest patches of the last num_versions major versions via Git CLI -def getLastSensorVersionsFromGitTagsCLI(current_version, num_versions): +def getLastSensorVersionsFromGitTagsCLI(num_versions): rawtags = cliOutputToTags(subprocess.check_output(["git", "tag", "--list"])) tags = filterGitCLITags(rawtags) - return getLatestTags(current_version, tags, num_versions) + return getLatestTags(tags, num_versions) diff --git a/scripts/ci/jobs/gke_version_compatibility_tests.py b/scripts/ci/jobs/gke_version_compatibility_tests.py index 5899a0368679a..785c3eb3b36cc 100755 --- a/scripts/ci/jobs/gke_version_compatibility_tests.py +++ b/scripts/ci/jobs/gke_version_compatibility_tests.py @@ -14,7 +14,7 @@ # don't use postgres os.environ["ROX_POSTGRES_DATASTORE"] = "false" -versions=getLastSensorVersionsFromGitTagsCLI(os.environ["MAIN_IMAGE_TAG"], 4) +versions=getLastSensorVersionsFromGitTagsCLI(4) gkecluster=GKECluster("qa-e2e-test") From fed49f7188ebad9d75995579352040aad5f3b62c Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 15:04:29 +0100 Subject: [PATCH 15/32] Change function names to underscores instead of camel case --- .openshift-ci/sensor_versions.py | 32 +++++++++---------- .../jobs/gke_version_compatibility_tests.py | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.openshift-ci/sensor_versions.py b/.openshift-ci/sensor_versions.py index 410b9c5a2915a..ca0bbede97c8c 100644 --- a/.openshift-ci/sensor_versions.py +++ b/.openshift-ci/sensor_versions.py @@ -8,33 +8,33 @@ import re import subprocess -def isReleaseTagGitCLI(version): +def is_release_tag_git_cli(version): return bool(re.search(r"^\d+\.\d+\.\d+$", version)) -def filterGitCLITags(rawtags): +def filter_git_cli_tags(rawtags): filteredtags = [] for t in rawtags: - if isReleaseTagGitCLI(t): + if is_release_tag_git_cli(t): filteredtags.append(t) return set(filteredtags) -def cliOutputToTags(stdoutput): +def cli_output_to_tags(stdoutput): separated = stdoutput.decode(encoding="utf-8").splitlines() return separated -def transformTagsToNumbers(tags): +def transform_tags_to_numbers(tags): numTags = [] for t in tags: - numTags.append(splitVersion(t)) + numTags.append(split_version(t)) return sorted(numTags) -def splitVersion(version): +def split_version(version): digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) -def getLatestTags(tags, num_versions): - numericaltags = transformTagsToNumbers(tags) - _,y,_ = splitVersion(makeImageTag()) +def get_latest_tags(tags, num_versions): + numericaltags = transform_tags_to_numbers(tags) + _,y,_ = split_version(make_image_tag()) latestversions = [] ycurr = y for tags in numericaltags[::-1]: @@ -47,11 +47,11 @@ def getLatestTags(tags, num_versions): ycurr-=1 return latestversions -def makeImageTag(): +def make_image_tag(): return subprocess.check_output(["make", "tag"]).decode(encoding="utf-8") -# getLastSensorVersionsFromGitTagsCLI gets the latest patches of the last num_versions major versions via Git CLI -def getLastSensorVersionsFromGitTagsCLI(num_versions): - rawtags = cliOutputToTags(subprocess.check_output(["git", "tag", "--list"])) - tags = filterGitCLITags(rawtags) - return getLatestTags(tags, num_versions) +# get_last_sensor_versions_from_git_tags_cli gets the latest patches of the last num_versions major versions via Git CLI +def get_last_sensor_versions_from_git_tags_cli(num_versions): + rawtags = cli_output_to_tags(subprocess.check_output(["git", "tag", "--list"])) + tags = filter_git_cli_tags(rawtags) + return get_latest_tags(tags, num_versions) diff --git a/scripts/ci/jobs/gke_version_compatibility_tests.py b/scripts/ci/jobs/gke_version_compatibility_tests.py index 785c3eb3b36cc..66a2c8a84e9b5 100755 --- a/scripts/ci/jobs/gke_version_compatibility_tests.py +++ b/scripts/ci/jobs/gke_version_compatibility_tests.py @@ -4,7 +4,7 @@ Run version compatibility tests """ import os -from sensor_versions import getLastSensorVersionsFromGitTagsCLI +from sensor_versions import get_last_sensor_versions_from_git_tags_cli from compatibility_test import make_compatibility_test_runner from clusters import GKECluster @@ -14,7 +14,7 @@ # don't use postgres os.environ["ROX_POSTGRES_DATASTORE"] = "false" -versions=getLastSensorVersionsFromGitTagsCLI(4) +versions=get_last_sensor_versions_from_git_tags_cli(4) gkecluster=GKECluster("qa-e2e-test") From 15681597d1edf0b23f23f9d44bf31ade8eb2c592 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 15:15:49 +0100 Subject: [PATCH 16/32] Split split_version into two functions --- .openshift-ci/sensor_versions.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.openshift-ci/sensor_versions.py b/.openshift-ci/sensor_versions.py index ca0bbede97c8c..4e6b528f5d19a 100644 --- a/.openshift-ci/sensor_versions.py +++ b/.openshift-ci/sensor_versions.py @@ -29,12 +29,15 @@ def transform_tags_to_numbers(tags): return sorted(numTags) def split_version(version): - digits = re.search(r"(\d+)\.(\d+)\.\D*(\d+)", version) + digits = re.search(r"(\d+)\.(\d+)\.(\d+)", version) return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) +def extract_y_from_main_image_tag(mainimagetag): + return int(re.search(r"\d+\.(\d+)", mainimagetag).group(1)) + def get_latest_tags(tags, num_versions): numericaltags = transform_tags_to_numbers(tags) - _,y,_ = split_version(make_image_tag()) + y = extract_y_from_main_image_tag(make_image_tag()) latestversions = [] ycurr = y for tags in numericaltags[::-1]: From 8f28b02c25e5357a65983635bbf2793379b4fdd4 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 15:20:43 +0100 Subject: [PATCH 17/32] Refactor and simplify using dict --- .openshift-ci/sensor_versions.py | 34 ++++++++------------------------ 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/.openshift-ci/sensor_versions.py b/.openshift-ci/sensor_versions.py index 4e6b528f5d19a..f8832d15ba891 100644 --- a/.openshift-ci/sensor_versions.py +++ b/.openshift-ci/sensor_versions.py @@ -7,6 +7,7 @@ import sys import re import subprocess +from collections import defaultdict def is_release_tag_git_cli(version): return bool(re.search(r"^\d+\.\d+\.\d+$", version)) @@ -22,33 +23,14 @@ def cli_output_to_tags(stdoutput): separated = stdoutput.decode(encoding="utf-8").splitlines() return separated -def transform_tags_to_numbers(tags): - numTags = [] - for t in tags: - numTags.append(split_version(t)) - return sorted(numTags) - -def split_version(version): - digits = re.search(r"(\d+)\.(\d+)\.(\d+)", version) - return int(digits.group(1)), int(digits.group(2)), int(digits.group(3)) - -def extract_y_from_main_image_tag(mainimagetag): - return int(re.search(r"\d+\.(\d+)", mainimagetag).group(1)) - def get_latest_tags(tags, num_versions): - numericaltags = transform_tags_to_numbers(tags) - y = extract_y_from_main_image_tag(make_image_tag()) - latestversions = [] - ycurr = y - for tags in numericaltags[::-1]: - if tags[1] < y-num_versions+1: - break - if tags[1] < ycurr: - ycurr = tags[1] - if tags[1] == ycurr: - latestversions.append(str(tags[0]) + "." + str(tags[1]) + "." + str(tags[2])) - ycurr-=1 - return latestversions + top_patch_version = defaultdict(int) + for t in tags: + [major, minor, patch] = t.split('.') + k = '.'.join([major, minor]) + top_patch_version[k] = max(top_patch_version[k], int(patch)) + top_major_versions = sorted(list(top_patch_version.keys()), reverse=True)[:num_versions] + return [t + '.' + str(top_patch_version[t]) for t in top_major_versions] def make_image_tag(): return subprocess.check_output(["make", "tag"]).decode(encoding="utf-8") From 793544e33a950f89adf24e10301a4f5c3b253470 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 15:36:21 +0100 Subject: [PATCH 18/32] Ensure that no sensor versions newer than central get tested --- .openshift-ci/sensor_versions.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.openshift-ci/sensor_versions.py b/.openshift-ci/sensor_versions.py index f8832d15ba891..da15a5ce93ec5 100644 --- a/.openshift-ci/sensor_versions.py +++ b/.openshift-ci/sensor_versions.py @@ -23,18 +23,23 @@ def cli_output_to_tags(stdoutput): separated = stdoutput.decode(encoding="utf-8").splitlines() return separated +def make_image_tag(): + return subprocess.check_output(["make", "tag"]).decode(encoding="utf-8") + +def extract_y_from_main_image_tag(mainimagetag): + return int(re.search(r"\d+\.(\d+)", mainimagetag).group(1)) + def get_latest_tags(tags, num_versions): + main_image_y = extract_y_from_main_image_tag(make_image_tag()) top_patch_version = defaultdict(int) for t in tags: [major, minor, patch] = t.split('.') k = '.'.join([major, minor]) - top_patch_version[k] = max(top_patch_version[k], int(patch)) + if (int(minor) <= main_image_y): + top_patch_version[k] = max(top_patch_version[k], int(patch)) top_major_versions = sorted(list(top_patch_version.keys()), reverse=True)[:num_versions] return [t + '.' + str(top_patch_version[t]) for t in top_major_versions] -def make_image_tag(): - return subprocess.check_output(["make", "tag"]).decode(encoding="utf-8") - # get_last_sensor_versions_from_git_tags_cli gets the latest patches of the last num_versions major versions via Git CLI def get_last_sensor_versions_from_git_tags_cli(num_versions): rawtags = cli_output_to_tags(subprocess.check_output(["git", "tag", "--list"])) From 0bce4cf995e561b0162b272954a3829e73bfffd7 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 15:41:21 +0100 Subject: [PATCH 19/32] Rename main script function to be less verbose --- .openshift-ci/sensor_versions.py | 4 ++-- scripts/ci/jobs/gke_version_compatibility_tests.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.openshift-ci/sensor_versions.py b/.openshift-ci/sensor_versions.py index da15a5ce93ec5..2211c02edc5bc 100644 --- a/.openshift-ci/sensor_versions.py +++ b/.openshift-ci/sensor_versions.py @@ -40,8 +40,8 @@ def get_latest_tags(tags, num_versions): top_major_versions = sorted(list(top_patch_version.keys()), reverse=True)[:num_versions] return [t + '.' + str(top_patch_version[t]) for t in top_major_versions] -# get_last_sensor_versions_from_git_tags_cli gets the latest patches of the last num_versions major versions via Git CLI -def get_last_sensor_versions_from_git_tags_cli(num_versions): +# get_latest_release_versions gets the latest patches of the last num_versions major versions via Git CLI +def get_latest_release_versions(num_versions): rawtags = cli_output_to_tags(subprocess.check_output(["git", "tag", "--list"])) tags = filter_git_cli_tags(rawtags) return get_latest_tags(tags, num_versions) diff --git a/scripts/ci/jobs/gke_version_compatibility_tests.py b/scripts/ci/jobs/gke_version_compatibility_tests.py index 66a2c8a84e9b5..8225bdf471360 100755 --- a/scripts/ci/jobs/gke_version_compatibility_tests.py +++ b/scripts/ci/jobs/gke_version_compatibility_tests.py @@ -4,7 +4,7 @@ Run version compatibility tests """ import os -from sensor_versions import get_last_sensor_versions_from_git_tags_cli +from sensor_versions import get_latest_release_versions from compatibility_test import make_compatibility_test_runner from clusters import GKECluster @@ -14,7 +14,7 @@ # don't use postgres os.environ["ROX_POSTGRES_DATASTORE"] = "false" -versions=get_last_sensor_versions_from_git_tags_cli(4) +versions=get_latest_release_versions(4) gkecluster=GKECluster("qa-e2e-test") From 6b8bf856a1f3a891dbad84a09655edb302477dea Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 16:06:07 +0100 Subject: [PATCH 20/32] Readd main function --- ...sensor_versions.py => get_latest_release_versions.py} | 9 ++++++++- scripts/ci/jobs/gke_version_compatibility_tests.py | 6 ++---- 2 files changed, 10 insertions(+), 5 deletions(-) rename .openshift-ci/{sensor_versions.py => get_latest_release_versions.py} (87%) diff --git a/.openshift-ci/sensor_versions.py b/.openshift-ci/get_latest_release_versions.py similarity index 87% rename from .openshift-ci/sensor_versions.py rename to .openshift-ci/get_latest_release_versions.py index 2211c02edc5bc..051991690f860 100644 --- a/.openshift-ci/sensor_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ -Returns the latest patches of the last 4 major versions +Returns the latest patches of the last n major versions """ import sys @@ -45,3 +45,10 @@ def get_latest_release_versions(num_versions): rawtags = cli_output_to_tags(subprocess.check_output(["git", "tag", "--list"])) tags = filter_git_cli_tags(rawtags) return get_latest_tags(tags, num_versions) + +def main(argv): + latestversions = get_latest_release_versions(int(argv[1])) + print("\n".join(latestversions)) + +if (__name__ == "__main__"): + main(sys.argv) diff --git a/scripts/ci/jobs/gke_version_compatibility_tests.py b/scripts/ci/jobs/gke_version_compatibility_tests.py index 8225bdf471360..d5fe936749b96 100755 --- a/scripts/ci/jobs/gke_version_compatibility_tests.py +++ b/scripts/ci/jobs/gke_version_compatibility_tests.py @@ -4,7 +4,7 @@ Run version compatibility tests """ import os -from sensor_versions import get_latest_release_versions +import get_latest_release_versions from compatibility_test import make_compatibility_test_runner from clusters import GKECluster @@ -14,12 +14,10 @@ # don't use postgres os.environ["ROX_POSTGRES_DATASTORE"] = "false" -versions=get_latest_release_versions(4) +versions=get_latest_release_versions.get_latest_release_versions(4) gkecluster=GKECluster("qa-e2e-test") for version in versions: os.environ["SENSOR_IMAGE_TAG"] = version make_compatibility_test_runner(cluster=gkecluster).run() - -print("stub for version compatibility tests") From 7bf3c446b5e75b21bbbdbb54cfeee5dc4545ece7 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 16:07:16 +0100 Subject: [PATCH 21/32] Add "--quiet" and "--no-print-directory" to make tag call --- .openshift-ci/get_latest_release_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index 051991690f860..8d66bbe4f3138 100644 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -24,7 +24,7 @@ def cli_output_to_tags(stdoutput): return separated def make_image_tag(): - return subprocess.check_output(["make", "tag"]).decode(encoding="utf-8") + return subprocess.check_output(["make", "--quiet", "--no-print-directory", "tag"]).decode(encoding="utf-8") def extract_y_from_main_image_tag(mainimagetag): return int(re.search(r"\d+\.(\d+)", mainimagetag).group(1)) From d43ffc0f04eff02013ef0023838c7266158cbdeb Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 16:09:42 +0100 Subject: [PATCH 22/32] Rename functions to make them shorter --- .openshift-ci/get_latest_release_versions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index 8d66bbe4f3138..b0b4968b5fa8b 100644 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -9,13 +9,13 @@ import subprocess from collections import defaultdict -def is_release_tag_git_cli(version): +def is_release_tag(version): return bool(re.search(r"^\d+\.\d+\.\d+$", version)) -def filter_git_cli_tags(rawtags): +def filter_tags(rawtags): filteredtags = [] for t in rawtags: - if is_release_tag_git_cli(t): + if is_release_tag(t): filteredtags.append(t) return set(filteredtags) @@ -43,7 +43,7 @@ def get_latest_tags(tags, num_versions): # get_latest_release_versions gets the latest patches of the last num_versions major versions via Git CLI def get_latest_release_versions(num_versions): rawtags = cli_output_to_tags(subprocess.check_output(["git", "tag", "--list"])) - tags = filter_git_cli_tags(rawtags) + tags = filter_tags(rawtags) return get_latest_tags(tags, num_versions) def main(argv): From 1021b41b5ee14b90b428db27ea8f73db09558914 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 16:11:14 +0100 Subject: [PATCH 23/32] Use is not None instead of bool() cast --- .openshift-ci/get_latest_release_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index b0b4968b5fa8b..aaf609b3a38c4 100644 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -10,7 +10,7 @@ from collections import defaultdict def is_release_tag(version): - return bool(re.search(r"^\d+\.\d+\.\d+$", version)) + return re.search(r"^\d+\.\d+\.\d+$", version) is not None def filter_tags(rawtags): filteredtags = [] From 963eebe7b849de1187393470e196d7e11fcf25bc Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 16:11:39 +0100 Subject: [PATCH 24/32] Remove unused variable --- .openshift-ci/get_latest_release_versions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index aaf609b3a38c4..891a4e2c76f0c 100644 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -20,8 +20,7 @@ def filter_tags(rawtags): return set(filteredtags) def cli_output_to_tags(stdoutput): - separated = stdoutput.decode(encoding="utf-8").splitlines() - return separated + return stdoutput.decode(encoding="utf-8").splitlines() def make_image_tag(): return subprocess.check_output(["make", "--quiet", "--no-print-directory", "tag"]).decode(encoding="utf-8") From 731d912ce14309081a9535daaaf10f8e4dd126d1 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 16:13:10 +0100 Subject: [PATCH 25/32] Shorten filter_tags function --- .openshift-ci/get_latest_release_versions.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index 891a4e2c76f0c..969affafcade9 100644 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -13,11 +13,7 @@ def is_release_tag(version): return re.search(r"^\d+\.\d+\.\d+$", version) is not None def filter_tags(rawtags): - filteredtags = [] - for t in rawtags: - if is_release_tag(t): - filteredtags.append(t) - return set(filteredtags) + return set([t for t in rawtags if is_release_tag(t)]) def cli_output_to_tags(stdoutput): return stdoutput.decode(encoding="utf-8").splitlines() From 04de0f999cc1e4a88639f3b6ad0b91587e6815e9 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 16:14:23 +0100 Subject: [PATCH 26/32] Inline cli_output_to_tags function --- .openshift-ci/get_latest_release_versions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index 969affafcade9..52486eda198ad 100644 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -15,9 +15,6 @@ def is_release_tag(version): def filter_tags(rawtags): return set([t for t in rawtags if is_release_tag(t)]) -def cli_output_to_tags(stdoutput): - return stdoutput.decode(encoding="utf-8").splitlines() - def make_image_tag(): return subprocess.check_output(["make", "--quiet", "--no-print-directory", "tag"]).decode(encoding="utf-8") @@ -37,7 +34,7 @@ def get_latest_tags(tags, num_versions): # get_latest_release_versions gets the latest patches of the last num_versions major versions via Git CLI def get_latest_release_versions(num_versions): - rawtags = cli_output_to_tags(subprocess.check_output(["git", "tag", "--list"])) + rawtags = subprocess.check_output(["git", "tag", "--list"]).decode(encoding="utf-8").splitlines() tags = filter_tags(rawtags) return get_latest_tags(tags, num_versions) From d8a469b5b4af963d0763996accbc0877b73c66e2 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 18:15:54 +0100 Subject: [PATCH 27/32] compare major and minor versions --- .openshift-ci/get_latest_release_versions.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index 52486eda198ad..45891a6bf7bb1 100644 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -18,16 +18,17 @@ def filter_tags(rawtags): def make_image_tag(): return subprocess.check_output(["make", "--quiet", "--no-print-directory", "tag"]).decode(encoding="utf-8") -def extract_y_from_main_image_tag(mainimagetag): - return int(re.search(r"\d+\.(\d+)", mainimagetag).group(1)) +def extract_x_y_from_main_image_tag(mainimagetag): + x_y = re.search(r"(\d+)\.(\d+)", mainimagetag) + return int(x_y.group(1)), int(x_y.group(2)) def get_latest_tags(tags, num_versions): - main_image_y = extract_y_from_main_image_tag(make_image_tag()) + central_major, central_minor = extract_x_y_from_main_image_tag(make_image_tag()) top_patch_version = defaultdict(int) for t in tags: [major, minor, patch] = t.split('.') k = '.'.join([major, minor]) - if (int(minor) <= main_image_y): + if (int(major) < central_major or (int(major) == central_major and int(minor) <= central_minor)): top_patch_version[k] = max(top_patch_version[k], int(patch)) top_major_versions = sorted(list(top_patch_version.keys()), reverse=True)[:num_versions] return [t + '.' + str(top_patch_version[t]) for t in top_major_versions] @@ -39,7 +40,9 @@ def get_latest_release_versions(num_versions): return get_latest_tags(tags, num_versions) def main(argv): - latestversions = get_latest_release_versions(int(argv[1])) + n = int(argv[1]) if len(argv)>1 else 4 + latestversions = get_latest_release_versions(n) + print(f"Last {n} versions:") print("\n".join(latestversions)) if (__name__ == "__main__"): From fea9ce42c3c19a8e4b123be5b3ab2e3980b18a9f Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 23:46:22 +0100 Subject: [PATCH 28/32] make script executable --- .openshift-ci/get_latest_release_versions.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .openshift-ci/get_latest_release_versions.py diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py old mode 100644 new mode 100755 From 653bccf10a6131a6f120a10129aadb39b82aeea1 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Mon, 14 Nov 2022 23:48:10 +0100 Subject: [PATCH 29/32] Constrain regex search more --- .openshift-ci/get_latest_release_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index 45891a6bf7bb1..78a8047035bbf 100755 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -19,7 +19,7 @@ def make_image_tag(): return subprocess.check_output(["make", "--quiet", "--no-print-directory", "tag"]).decode(encoding="utf-8") def extract_x_y_from_main_image_tag(mainimagetag): - x_y = re.search(r"(\d+)\.(\d+)", mainimagetag) + x_y = re.search(r"^(\d+)\.(\d+)", mainimagetag) return int(x_y.group(1)), int(x_y.group(2)) def get_latest_tags(tags, num_versions): From 44be8729508a8a3da5025e17d22072a6532343b9 Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Tue, 15 Nov 2022 00:11:26 +0100 Subject: [PATCH 30/32] Split get_latest_tags into two functions --- .openshift-ci/get_latest_release_versions.py | 24 ++++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index 78a8047035bbf..c6b595bbaec05 100755 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -15,6 +15,15 @@ def is_release_tag(version): def filter_tags(rawtags): return set([t for t in rawtags if is_release_tag(t)]) +def reduce_tags_to_latest_patch(tags): + top_patch_version = defaultdict(int) + for t in tags: + [major, minor, patch] = t.split('.') + k = '.'.join([major, minor]) + top_patch_version[k] = max(top_patch_version[k], int(patch)) + top_major_versions = sorted(list(top_patch_version.keys()), reverse=True) + return [t + '.' + str(top_patch_version[t]) for t in top_major_versions] + def make_image_tag(): return subprocess.check_output(["make", "--quiet", "--no-print-directory", "tag"]).decode(encoding="utf-8") @@ -22,22 +31,17 @@ def extract_x_y_from_main_image_tag(mainimagetag): x_y = re.search(r"^(\d+)\.(\d+)", mainimagetag) return int(x_y.group(1)), int(x_y.group(2)) -def get_latest_tags(tags, num_versions): +def get_latest_n_tags(tags, num_versions): central_major, central_minor = extract_x_y_from_main_image_tag(make_image_tag()) - top_patch_version = defaultdict(int) - for t in tags: - [major, minor, patch] = t.split('.') - k = '.'.join([major, minor]) - if (int(major) < central_major or (int(major) == central_major and int(minor) <= central_minor)): - top_patch_version[k] = max(top_patch_version[k], int(patch)) - top_major_versions = sorted(list(top_patch_version.keys()), reverse=True)[:num_versions] - return [t + '.' + str(top_patch_version[t]) for t in top_major_versions] + tags_older_than_central = [t for t in tags if (int(t.split('.')[0]) < central_major or (int(t.split('.')[0]) == central_major and int(t.split('.')[1]) <= central_minor))] + return tags_older_than_central[:num_versions] # get_latest_release_versions gets the latest patches of the last num_versions major versions via Git CLI def get_latest_release_versions(num_versions): rawtags = subprocess.check_output(["git", "tag", "--list"]).decode(encoding="utf-8").splitlines() tags = filter_tags(rawtags) - return get_latest_tags(tags, num_versions) + latest_patch_tags = reduce_tags_to_latest_patch(tags) + return get_latest_n_tags(latest_patch_tags, num_versions) def main(argv): n = int(argv[1]) if len(argv)>1 else 4 From cb632867974f299edfd843051494fd2d8abae82c Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Tue, 15 Nov 2022 15:58:34 +0100 Subject: [PATCH 31/32] Import only the function get_latest_release_versions instead of the whole module --- scripts/ci/jobs/gke_version_compatibility_tests.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/ci/jobs/gke_version_compatibility_tests.py b/scripts/ci/jobs/gke_version_compatibility_tests.py index d5fe936749b96..fea9267bc497c 100755 --- a/scripts/ci/jobs/gke_version_compatibility_tests.py +++ b/scripts/ci/jobs/gke_version_compatibility_tests.py @@ -4,7 +4,7 @@ Run version compatibility tests """ import os -import get_latest_release_versions +from get_latest_release_versions import get_latest_release_versions from compatibility_test import make_compatibility_test_runner from clusters import GKECluster @@ -14,10 +14,12 @@ # don't use postgres os.environ["ROX_POSTGRES_DATASTORE"] = "false" -versions=get_latest_release_versions.get_latest_release_versions(4) +versions=get_latest_release_versions(4) +print("\n".join(versions)) + gkecluster=GKECluster("qa-e2e-test") -for version in versions: - os.environ["SENSOR_IMAGE_TAG"] = version - make_compatibility_test_runner(cluster=gkecluster).run() +#for version in versions: +# os.environ["SENSOR_IMAGE_TAG"] = version +# make_compatibility_test_runner(cluster=gkecluster).run() From f9b604916a878d960dab792a5936571bdd5834ab Mon Sep 17 00:00:00 2001 From: Jan Schnathmeier Date: Tue, 15 Nov 2022 15:59:36 +0100 Subject: [PATCH 32/32] rename mainimagetag to tag --- .openshift-ci/get_latest_release_versions.py | 4 ++-- scripts/ci/jobs/gke_version_compatibility_tests.py | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.openshift-ci/get_latest_release_versions.py b/.openshift-ci/get_latest_release_versions.py index c6b595bbaec05..91c74b1ab504e 100755 --- a/.openshift-ci/get_latest_release_versions.py +++ b/.openshift-ci/get_latest_release_versions.py @@ -27,8 +27,8 @@ def reduce_tags_to_latest_patch(tags): def make_image_tag(): return subprocess.check_output(["make", "--quiet", "--no-print-directory", "tag"]).decode(encoding="utf-8") -def extract_x_y_from_main_image_tag(mainimagetag): - x_y = re.search(r"^(\d+)\.(\d+)", mainimagetag) +def extract_x_y_from_main_image_tag(tag): + x_y = re.search(r"^(\d+)\.(\d+)", tag) return int(x_y.group(1)), int(x_y.group(2)) def get_latest_n_tags(tags, num_versions): diff --git a/scripts/ci/jobs/gke_version_compatibility_tests.py b/scripts/ci/jobs/gke_version_compatibility_tests.py index fea9267bc497c..5fd110e350f07 100755 --- a/scripts/ci/jobs/gke_version_compatibility_tests.py +++ b/scripts/ci/jobs/gke_version_compatibility_tests.py @@ -15,11 +15,9 @@ os.environ["ROX_POSTGRES_DATASTORE"] = "false" versions=get_latest_release_versions(4) -print("\n".join(versions)) - gkecluster=GKECluster("qa-e2e-test") -#for version in versions: -# os.environ["SENSOR_IMAGE_TAG"] = version -# make_compatibility_test_runner(cluster=gkecluster).run() +for version in versions: + os.environ["SENSOR_IMAGE_TAG"] = version + make_compatibility_test_runner(cluster=gkecluster).run()