From c6835c713cfb1c0fdd535d4e73ac79c6b4b40ce0 Mon Sep 17 00:00:00 2001 From: dnichol Date: Wed, 29 Mar 2023 10:39:43 +0100 Subject: [PATCH 001/146] Fix to include externalName in update --- examples/update_usergroups_from_csv.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/update_usergroups_from_csv.py b/examples/update_usergroups_from_csv.py index 7e2260e2..07df9de8 100644 --- a/examples/update_usergroups_from_csv.py +++ b/examples/update_usergroups_from_csv.py @@ -102,6 +102,8 @@ def update_user_group(hub_client, existing_name, new_name, user_group): # Update the name. user_group['name'] = new_name + #if user_group.get('externalGroupName') and user['externalGroupName'] == name: + user_group['externalName'] = new_name logging.info(f"Updating user group {existing_name} to {user_group['name']} for user group {user_group_url}") hub_client.session.put(user_group_url, json=user_group) From 5db21d26e9670270eab17140612624f45ddc27c8 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 21 Jun 2023 19:47:21 -0400 Subject: [PATCH 002/146] full scan --- .../multi-image/manage_project_structure.py | 258 ++++++++++++++++++ 1 file changed, 258 insertions(+) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 8b137891..0a963d16 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -1 +1,259 @@ +#!/usr/bin/env python3 +''' +Copyright (C) 2023 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +This script will scan a multi-container project into +hierarchical structure. + +Project Name + Project Version + Subproject Name + Subproject Vesrsion + Base Image + Base Image Version + Add on Image + Add on Image Version + +usage: python3 manage_project_structure.py [-h] -u BASE_URL -t TOKEN_FILE [-pg PROJECT_GROUP] -p PROJECT_NAME -pv VERSION_NAME + [-sp SUBPROJECT_LIST] [-nv] [-rm] [--dry-run] + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -pg PROJECT_GROUP, --project_group PROJECT_GROUP + Project Group to be used + -p PROJECT_NAME, --project-name PROJECT_NAME + Project Name + -pv VERSION_NAME, --version-name VERSION_NAME + Project Version Name + -sp SUBPROJECT_LIST, --subproject-list SUBPROJECT_LIST + List of subprojects to generate with subproject:container:tag + -nv, --no-verify Disable TLS certificate verification + -rm, --remove Remove project structure with all subprojects (DANGEROUS!) + --dry-run Create structure only, do not execute scans + +Subprojects ae specified as subproject:[container]:[tag] +if container name omited it will be set to subproject +if tag omited it would be set to 'latest' + +Container image name scanned will be written into project version nickname field + + + +''' + +import argparse +import json +import logging +import sys +import arrow + +from blackduck import Client +from pprint import pprint,pformat + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.INFO) +logging.getLogger("urllib3").setLevel(logging.INFO) +logging.getLogger("blackduck").setLevel(logging.INFO) + +def remove_project_structure(project_name, version_name): + project = find_project_by_name(project_name) + if not project: + logging.info(f"Project {project_name} does not exist.") + return + num_versions = bd.get_resource('versions', project, items=False)['totalCount'] + version = find_project_version_by_name(project,version_name) + if not version: + logging.info(f"Project {project_name} with version {version_name} does not exist.") + return + components = [ + c for c in bd.get_resource('components',version) if c['componentType'] == "SUB_PROJECT" + ] + logging.info(f"Project {project_name}:{version_name} has {len(components)} subprojects") + for component in components: + component_name = component['componentName'] + component_version_name = component['componentVersionName'] + logging.info(f"Removing subproject {component_name} from {project_name}:{version_name}") + component_url = component['_meta']['href'] + response = bd.session.delete(component_url) + logging.info(f"Operation completed with {response}") + remove_project_structure(component_name, component_version_name) + logging.info(f"Removing {project_name}:{version_name}") + if num_versions > 1: + response = bd.session.delete(version['_meta']['href']) + else: + response = bd.session.delete(project['_meta']['href']) + logging.info(f"Operation completed with {response}") + +def find_or_create_project_group(group_name): + url = '/api/project-groups' + params = { + 'q': [f"name:{group_name}"] + } + groups = [p for p in bd.get_items(url, params=params) if p['name'] == group_name] + if len(groups) == 0: + headers = { + 'Accept': 'application/vnd.blackducksoftware.project-detail-5+json', + 'Content-Type': 'application/vnd.blackducksoftware.project-detail-5+json' + } + data = { + 'name': group_name + } + response = bd.session.post(url, headers=headers, json=data) + return response.headers['Location'] + else: + return groups[0]['_meta']['href'] + +def find_project_by_name(project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + +def create_project_version(project_name,version_name,args, nickname = None): + version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name} + if nickname: + version_data['nickname'] = nickname + url = '/api/projects' + project = find_project_by_name(project_name) + if project: + data = version_data + url = project['_meta']['href'] + '/versions' + else: + data = {"name": project_name, + "projectGroup": find_or_create_project_group(args.project_group), + "versionRequest": version_data} + return bd.session.post(url, json=data) + +def add_component_to_version_bom(child_version, version): + url = version['_meta']['href'] + '/components' + data = { 'component': child_version['_meta']['href']} + return bd.session.post(url, json=data) + +def create_and_add_child_projects(version, args): + version_url = version['_meta']['href'] + '/components' + for child_spec in [x.split(':') for x in args.subproject_list.split(",")]: + i = iter(child_spec) + child = next(i) + repo = next(i, child) + tag = next(i,'latest') + container_spec = f"{repo}:{tag}" + scan_param = {'image': container_spec, 'project': child, 'version': args.version_name} + project = find_project_by_name(child) + if project: + version = find_project_version_by_name(project,args.version_name) + if version: + logging.error(f"Child project {project['name']} with version {args.version_name} exists.") + return + response = create_project_version(child,args.version_name, args, nickname=container_spec) + logging.info(f"Creating project {child} : {args.version_name} completed with {response}") + if response.ok: + child_version = find_project_version_by_name(find_project_by_name(child),args.version_name) + child_version_url = child_version['_meta']['href'] + response = bd.session.post(version_url,json={'component': child_version_url}) + logging.info(f"Adding {child} : {args.version_name} to parent project completed with {response}") + scan_params.append(scan_param) + +def create_project_structure(args): + project = find_project_by_name(args.project_name) + logging.info(f"Project {args.project_name} located") + if project: + version = find_project_version_by_name(project,args.version_name) + if version: + logging.error(f"Project {project['name']} with version {args.version_name} exists.") + sys.exit(1) + response = create_project_version(args.project_name,args.version_name,args) + if response.ok: + version = find_project_version_by_name(find_project_by_name(args.project_name),args.version_name) + logging.info(f"Project {args.project_name} : {args.version_name} created") + create_and_add_child_projects(version, args) + +def scan_container_images(scan_params): + from scan_docker_image_lite import scan_container_image + for params in scan_params: + scan_container_image( + params['image'], + None, + None, + None, + params['project'], + params['version'], + (f"--detect.parent.project.name={params['project']} " + f"--detect.parent.project.version.name={params['version']} " + f"--detect.project.version.nickname={params['image']}") + ) + + +def parse_command_args(): + + parser = argparse.ArgumentParser("python3 manage_project_structure.py") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-pg", "--project_group", required=False, default='Multi-Image', help="Project Group to be used") + parser.add_argument("-p", "--project-name", required=True, help="Project Name") + parser.add_argument("-pv", "--version-name", required=True, help="Project Version Name") + parser.add_argument("-sp", "--subproject-list", required=False, help="List of subprojects to generate with subproject:container:tag") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-rm", "--remove", action='store_true', required=False, help="Remove project structure with all subprojects (DANGEROUS!)") + parser.add_argument("--dry-run", action='store_true', required=False, help="Create structure only, do not execute scans") + return parser.parse_args() + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + global bd + global scan_params + scan_params = [] + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + logging.info(f"{args}") + + if args.remove: + remove_project_structure(args.project_name, args.version_name) + else: + create_project_structure(args) + if args.dry_run: + logging.info(f"{pformat(scan_params)}") + else: + logging.info("Now execution scans") + scan_container_images(scan_params) + + +if __name__ == "__main__": + sys.exit(main()) From b1ea6c8fbebd3d82474dbfff35c264c4a0b1a47d Mon Sep 17 00:00:00 2001 From: Thomas Graziadei Date: Wed, 8 Sep 2021 15:05:48 +0200 Subject: [PATCH 003/146] Preserve newline character in remediation comments fields. --- examples/client/get_project_vulnerabilites_as_csv.py | 2 +- examples/vuln_batch_remediation.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/client/get_project_vulnerabilites_as_csv.py b/examples/client/get_project_vulnerabilites_as_csv.py index e3be492d..8dfe89a8 100644 --- a/examples/client/get_project_vulnerabilites_as_csv.py +++ b/examples/client/get_project_vulnerabilites_as_csv.py @@ -27,7 +27,7 @@ ) def strip_newline(str): - return str.replace('\r', '').replace('\n', ' ') + return str.replace('\r', '').replace('\n', '\\n') def match_component(selected_components, component): if (len(selected_components) == 0): diff --git a/examples/vuln_batch_remediation.py b/examples/vuln_batch_remediation.py index 2b006bb1..2b1b1264 100644 --- a/examples/vuln_batch_remediation.py +++ b/examples/vuln_batch_remediation.py @@ -127,7 +127,7 @@ def set_vulnerablity_remediation(hub, vuln, remediation_status, remediation_comm url = vuln['_meta']['href'] update={} update['remediationStatus'] = remediation_status - update['comment'] = remediation_comment + update['comment'] = remediation_comment.replace('\\n','\n') response = hub.execute_put(url, data=update) return response @@ -283,4 +283,4 @@ def main(argv=None): # IGNORE:C0111 return 0 if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) From 83bfd61ad4168f3c11c246a7606891e18209aa09 Mon Sep 17 00:00:00 2001 From: Manfred Rudigier Date: Wed, 12 Jul 2023 11:31:29 +0200 Subject: [PATCH 004/146] vuln_batch_remediation.py: Auto detect CSV delimiter. --- examples/vuln_batch_remediation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/vuln_batch_remediation.py b/examples/vuln_batch_remediation.py index 2b1b1264..0171740e 100644 --- a/examples/vuln_batch_remediation.py +++ b/examples/vuln_batch_remediation.py @@ -84,7 +84,9 @@ def load_remediation_input(remediation_file): with open(remediation_file, mode='r', encoding="utf-8") as infile: - reader = csv.reader(infile) + dialect = csv.Sniffer().sniff(infile.read(), delimiters=';,') + infile.seek(0) + reader = csv.reader(infile, dialect) #return {rows[0]:[rows[1],rows[2]] for rows in reader} return {rows[0]:rows[1:] for rows in reader} From 6f299a84bdde06ad741ea182cfe4512170ee0496 Mon Sep 17 00:00:00 2001 From: Manfred Rudigier Date: Wed, 12 Jul 2023 11:34:41 +0200 Subject: [PATCH 005/146] vuln_batch_remediation.py: Fix broken script by only requesting 1000 items. This makes the script work again with newer Blackduck versions. --- blackduck/Vulnerabilities.py | 1 + examples/vuln_batch_remediation.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/blackduck/Vulnerabilities.py b/blackduck/Vulnerabilities.py index fe3d665c..b9f4929d 100644 --- a/blackduck/Vulnerabilities.py +++ b/blackduck/Vulnerabilities.py @@ -28,6 +28,7 @@ def get_vulnerable_bom_components(self, version_obj, limit=9999): param_string = self._get_parameter_string({'limit': limit}) url = "{}{}".format(url, param_string) response = self.execute_get(url, custom_headers=custom_headers) + response.raise_for_status() return response.json() # TODO: Remove or refactor this diff --git a/examples/vuln_batch_remediation.py b/examples/vuln_batch_remediation.py index 0171740e..79c8ced8 100644 --- a/examples/vuln_batch_remediation.py +++ b/examples/vuln_batch_remediation.py @@ -273,9 +273,9 @@ def main(argv=None): # IGNORE:C0111 exclusion_data = None - # Retrieve the vulnerabiltites for the project version - vulnerable_components = hub.get_vulnerable_bom_components(version) + # Retrieve the vulnerabiltites for the project version. Newer API versions only allow 1000 items at most. + vulnerable_components = hub.get_vulnerable_bom_components(version, 1000) process_vulnerabilities(hub, vulnerable_components, remediation_data, exclusion_data, dry_run) return 0 From 32f922cfa3e4ade533a2ac5cbad496c592994c32 Mon Sep 17 00:00:00 2001 From: Manfred Rudigier Date: Wed, 12 Jul 2023 11:36:57 +0200 Subject: [PATCH 006/146] vuln_batch_remediation.py: Allow to overwrite existing remediations. This allows to bulk update many of the existing ones via CSV files. --- examples/vuln_batch_remediation.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/examples/vuln_batch_remediation.py b/examples/vuln_batch_remediation.py index 79c8ced8..a6cfcb5e 100644 --- a/examples/vuln_batch_remediation.py +++ b/examples/vuln_batch_remediation.py @@ -97,7 +97,7 @@ def remediation_is_valid(vuln, remediation_data): if vulnerability_name in remediation_data.keys(): remediation = remediation_data[vulnerability_name] - if (remediation_status == remediation[0] and remediation_comment == remediation[1]): + if (remediation_status == remediation[0] and remediation_comment == remediation[1].replace('\\n','\n')): return None return remediation_data[vulnerability_name] else: @@ -133,7 +133,7 @@ def set_vulnerablity_remediation(hub, vuln, remediation_status, remediation_comm response = hub.execute_put(url, data=update) return response -def process_vulnerabilities(hub, vulnerable_components, remediation_data=None, exclusion_data=None, dry_run=False): +def process_vulnerabilities(hub, vulnerable_components, remediation_data=None, exclusion_data=None, dry_run=False, overwrite_existing=False): if (dry_run): print(f"Opening dry run output file: {dry_run}") @@ -144,8 +144,8 @@ def process_vulnerabilities(hub, vulnerable_components, remediation_data=None, e print('"Component Name","Component Version","CVE","Reason","Remeidation Status","HTTP response code"') for vuln in vulnerable_components['items']: - if vuln['vulnerabilityWithRemediation']['remediationStatus'] == "NEW": - remediation_action = None + if overwrite_existing or vuln['vulnerabilityWithRemediation']['remediationStatus'] == "NEW": + remediation_action = None exclusion_action = None if (remediation_data): @@ -166,8 +166,7 @@ def process_vulnerabilities(hub, vulnerable_components, remediation_data=None, e if (remediation_action): if (dry_run): - remediation_action.insert(0, vuln['vulnerabilityWithRemediation']['vulnerabilityName']) - csv_writer.writerow(remediation_action) + csv_writer.writerow([vuln['vulnerabilityWithRemediation']['vulnerabilityName']] + remediation_action) else: resp = set_vulnerablity_remediation(hub, vuln, remediation_action[0],remediation_action[1]) count += 1 @@ -220,6 +219,7 @@ def main(argv=None): # IGNORE:C0111 parser.add_argument("--cve-remediation-list-custom-field-label", default='CVE Remediation List', help='Label of Custom Field on Black Duck that contains remeidation list file name') parser.add_argument("--origin-exclusion-list-custom-field-label", default='Origin Exclusion List', help='Label of Custom Field on Black Duck that containts origin exclusion list file name') parser.add_argument('-V', '--version', action='version', version=program_version_message) + parser.add_argument("--overwrite-existing", dest='overwrite_existing', action="store_true", help='By default only NEW vulnerabilities are remediated. Enabling this flag will update all vulnerabilities.') # Process arguments args = parser.parse_args() @@ -233,6 +233,7 @@ def main(argv=None): # IGNORE:C0111 #dry_run = args.dry_run #dry_run_output = args.dry_run_output dry_run = args.dry_run + overwrite_existing = args.overwrite_existing print(args.dry_run) message = f"{program_version_message}\n\n Project: {projectname}\n Version: {projectversion}\n Process origin exclusion list: {process_origin_exclulsion}\n Process CVE remediation list: {process_cve_remediation}" @@ -276,8 +277,9 @@ def main(argv=None): # IGNORE:C0111 # Retrieve the vulnerabiltites for the project version. Newer API versions only allow 1000 items at most. vulnerable_components = hub.get_vulnerable_bom_components(version, 1000) - process_vulnerabilities(hub, vulnerable_components, remediation_data, exclusion_data, dry_run) - + + process_vulnerabilities(hub, vulnerable_components, remediation_data, exclusion_data, dry_run, overwrite_existing) + return 0 except Exception: ### handle keyboard interrupt ### From 6903228664aa60b97fec7e44324ba78355050420 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 12 Jul 2023 21:19:31 -0400 Subject: [PATCH 007/146] cloning --- .../client/multi-image/manage_project_structure.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 0a963d16..2cb43f6a 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -175,6 +175,8 @@ def create_and_add_child_projects(version, args): tag = next(i,'latest') container_spec = f"{repo}:{tag}" scan_param = {'image': container_spec, 'project': child, 'version': args.version_name} + if args.clone_from: + scan_param['clone_from'] = args.clone_from project = find_project_by_name(child) if project: version = find_project_version_by_name(project,args.version_name) @@ -207,6 +209,12 @@ def create_project_structure(args): def scan_container_images(scan_params): from scan_docker_image_lite import scan_container_image for params in scan_params: + detect_options = (f"--detect.parent.project.name={params['project']} " + f"--detect.parent.project.version.name={params['version']} " + f"--detect.project.version.nickname={params['image']}") + clone_from = params.get('clone_from', None) + if clone_from: + detect_options += f" --detect.clone.project.version.name={clone_from}" scan_container_image( params['image'], None, @@ -214,9 +222,7 @@ def scan_container_images(scan_params): None, params['project'], params['version'], - (f"--detect.parent.project.name={params['project']} " - f"--detect.parent.project.version.name={params['version']} " - f"--detect.project.version.nickname={params['image']}") + detect_options ) @@ -231,6 +237,7 @@ def parse_command_args(): parser.add_argument("-sp", "--subproject-list", required=False, help="List of subprojects to generate with subproject:container:tag") parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") parser.add_argument("-rm", "--remove", action='store_true', required=False, help="Remove project structure with all subprojects (DANGEROUS!)") + parser.add_argument("--clone-from", required=False, help="Main project version to use as template for cloning") parser.add_argument("--dry-run", action='store_true', required=False, help="Create structure only, do not execute scans") return parser.parse_args() From 4e29a3c1219da2640314800423e1b995ab652623 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 12 Jul 2023 21:19:39 -0400 Subject: [PATCH 008/146] cloning --- examples/client/multi-image/scan_docker_image_lite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index ef06613c..442737fb 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -358,7 +358,8 @@ def adorn_extra_options(self, layer): option_to_adorn = '--detect.clone.project.version.name=' for option in self.extra_options: if option.startswith(option_to_adorn): - result.append(option.rstrip() + "_" + layer['group_name']) + # result.append(option.rstrip() + "_" + layer['group_name']) + result.append(option.rstrip()) else: result.append(option) return result From ea086fd26fda663c3aa270a10f1e117a08f8bf69 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 13 Jul 2023 13:14:31 -0400 Subject: [PATCH 009/146] cloning --- examples/client/multi-image/manage_project_structure.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 2cb43f6a..2632dacb 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -34,7 +34,7 @@ Add on Image Version usage: python3 manage_project_structure.py [-h] -u BASE_URL -t TOKEN_FILE [-pg PROJECT_GROUP] -p PROJECT_NAME -pv VERSION_NAME - [-sp SUBPROJECT_LIST] [-nv] [-rm] [--dry-run] + [-sp SUBPROJECT_LIST] [-nv] [-rm] [--clone-from CLONE_FROM] [--dry-run] options: -h, --help show this help message and exit @@ -52,6 +52,8 @@ List of subprojects to generate with subproject:container:tag -nv, --no-verify Disable TLS certificate verification -rm, --remove Remove project structure with all subprojects (DANGEROUS!) + --clone-from CLONE_FROM + Main project version to use as template for cloning --dry-run Create structure only, do not execute scans Subprojects ae specified as subproject:[container]:[tag] From 2c3a694410045c6af9fa0e0a5a65629e0e7cbd21 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 13 Jul 2023 13:45:18 -0400 Subject: [PATCH 010/146] shell example --- examples/client/multi-image/generate-clone.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 examples/client/multi-image/generate-clone.sh diff --git a/examples/client/multi-image/generate-clone.sh b/examples/client/multi-image/generate-clone.sh new file mode 100644 index 00000000..7337da2e --- /dev/null +++ b/examples/client/multi-image/generate-clone.sh @@ -0,0 +1,15 @@ +# Generate baseline project +# +# Project - container mapping + +SUBPROJECTS="\ +dashboard-ui:testcontainer:2.4,\ +docs-ui:testcontainer:2.4,\ +le:testcontainer:2.4,\ +login-app-ui:testcontainer:2.4,\ +nlv:testcontainer:2.4" + +COMMAND="python3 examples/client/multi-image/manage_project_structure.py" + +$COMMAND -u $BD_URL -t token -nv -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ + From 7ee264e8b81a7bd3c778239a7c886cd5960ae714 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 9 Aug 2023 17:30:11 -0400 Subject: [PATCH 011/146] fixed re-scan functionality --- .../multi-image/manage_project_structure.py | 93 +++++++++++++++---- 1 file changed, 76 insertions(+), 17 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 2632dacb..28467fac 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -80,6 +80,8 @@ logging.getLogger("urllib3").setLevel(logging.INFO) logging.getLogger("blackduck").setLevel(logging.INFO) +strict = False + def remove_project_structure(project_name, version_name): project = find_project_by_name(project_name) if not project: @@ -109,6 +111,33 @@ def remove_project_structure(project_name, version_name): response = bd.session.delete(project['_meta']['href']) logging.info(f"Operation completed with {response}") +def remove_codelocations_recursively(version): + components = bd.get_resource('components', version) + subprojects = [x for x in components if x['componentType'] == 'SUB_PROJECT'] + logging.info(f"Found {len(subprojects)} subprojects") + unmap_all_codelocations(version) + for subproject in subprojects: + subproject_name = subproject['componentName'] + subproject_version_name = subproject['componentVersionName'] + project = find_project_by_name(subproject_name) + if not project: + logging.info(f"Project {subproject_name} does not exist.") + return + subproject_version = find_project_version_by_name(project, subproject_version_name) + if not subproject_version: + logging.info(f"Project {subproject_name} with version {subversion_name} does not exist.") + return + remove_codelocations_recursively(subproject_version) + +def unmap_all_codelocations(version): + codelocations = bd.get_resource('codelocations',version) + for codelocation in codelocations: + logging.info(f"Unmapping codelocation {codelocation['name']}") + codelocation['mappedProjectVersion'] = "" + response = bd.session.put(codelocation['_meta']['href'], json=codelocation) + pprint (response) + + def find_or_create_project_group(group_name): url = '/api/project-groups' params = { @@ -183,16 +212,30 @@ def create_and_add_child_projects(version, args): if project: version = find_project_version_by_name(project,args.version_name) if version: - logging.error(f"Child project {project['name']} with version {args.version_name} exists.") - return - response = create_project_version(child,args.version_name, args, nickname=container_spec) - logging.info(f"Creating project {child} : {args.version_name} completed with {response}") - if response.ok: - child_version = find_project_version_by_name(find_project_by_name(child),args.version_name) - child_version_url = child_version['_meta']['href'] - response = bd.session.post(version_url,json={'component': child_version_url}) - logging.info(f"Adding {child} : {args.version_name} to parent project completed with {response}") - scan_params.append(scan_param) + if strict: + logging.error(f"Child project {project['name']} with version {args.version_name} exists.") + sys.exit(1) + else: + logging.info(f"Child project {project['name']} with version {args.version_name} found.") + logging.info(f"Recursively removing codelocations for {project['name']} with version {args.version_name} ") + remove_codelocations_recursively(version) + else: + response = create_project_version(child,args.version_name, args, nickname=container_spec) + logging.info(f"Creating project {child} : {args.version_name} completed with {response}") + if response.ok: + child_version = find_project_version_by_name(find_project_by_name(child),args.version_name) + child_version_url = child_version['_meta']['href'] + response = bd.session.post(version_url,json={'component': child_version_url}) + logging.info(f"Adding {child} : {args.version_name} to parent project completed with {response}") + else: + response = create_project_version(child,args.version_name, args, nickname=container_spec) + logging.info(f"Creating project {child} : {args.version_name} completed with {response}") + if response.ok: + child_version = find_project_version_by_name(find_project_by_name(child),args.version_name) + child_version_url = child_version['_meta']['href'] + response = bd.session.post(version_url,json={'component': child_version_url}) + logging.info(f"Adding {child} : {args.version_name} to parent project completed with {response}") + scan_params.append(scan_param) def create_project_structure(args): project = find_project_by_name(args.project_name) @@ -200,13 +243,29 @@ def create_project_structure(args): if project: version = find_project_version_by_name(project,args.version_name) if version: - logging.error(f"Project {project['name']} with version {args.version_name} exists.") - sys.exit(1) - response = create_project_version(args.project_name,args.version_name,args) - if response.ok: - version = find_project_version_by_name(find_project_by_name(args.project_name),args.version_name) - logging.info(f"Project {args.project_name} : {args.version_name} created") - create_and_add_child_projects(version, args) + if strict: + logging.error(f"Project {project['name']} with version {args.version_name} exists.") + sys.exit(1) + else: + logging.info(f"Found Project {project['name']} with version {args.version_name}.") + else: + response = create_project_version(args.project_name,args.version_name,args) + if response.ok: + version = find_project_version_by_name(find_project_by_name(args.project_name),args.version_name) + logging.info(f"Project {args.project_name} : {args.version_name} created") + else: + logging.info(f"Failed to create Project {args.project_name} : {args.version_name} created") + sys.exit(1) + else: + response = create_project_version(args.project_name,args.version_name,args) + if response.ok: + version = find_project_version_by_name(find_project_by_name(args.project_name),args.version_name) + logging.info(f"Project {args.project_name} : {args.version_name} created") + else: + logging.info(f"Failed to create Project {args.project_name} : {args.version_name} created") + sys.exit(1) + logging.info(f"Checking/Adding subprojects to {args.project_name} : {version['versionName']}") + create_and_add_child_projects(version, args) def scan_container_images(scan_params): from scan_docker_image_lite import scan_container_image From 1f8c22d5a9d9bff734efbd88f981d6bdcf34ef02 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Tue, 15 Aug 2023 11:23:53 -0400 Subject: [PATCH 012/146] Initial code for SPDX import/parse tool --- examples/client/parse_spdx.py | 308 ++++++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 examples/client/parse_spdx.py diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py new file mode 100644 index 00000000..9e16f98c --- /dev/null +++ b/examples/client/parse_spdx.py @@ -0,0 +1,308 @@ +''' +Created on August 15, 2023 +@author: swright + +##################### DISCLAIMER ########################## +## This script was created for a specific purpose and ## +## SHOULD NOT BE USED as a general purpose utility. ## +## For general purpose utility use ## +## /examples/client/generate_sbom.py ## +########################################################### + +Copyright (C) 2023 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +This script will parse a provided SPDX file and import the SBOM to the +specified Project Name and Project Version. + +Then it will search each component specified in the SPDX file to determine +if the component was succesfully imported. Any missing components will be +added as a custom component and then added to the Project+Verion's BOM. + +Requirements + +- python3 version 3.8 or newer recommended +- the following packages are used by the script and should be installed + prior to use: + argparse + blackduck + sys + logging + time + json + pprint + spdx_tools + +- Blackduck instance +- API token with sufficient privileges to perform project version phase + change. + +Install python packages with the following command: + + pip3 install argparse blackduck sys logging time json spdx_tools + +usage: parse_spdx.py [-h] --base-url BASE_URL --token-file TOKEN_FILE --spdx-file SPDX_FILE --out-file OUT_FILE --project PROJECT_NAME --version VERSION_NAME [--no-verify] + +Parse SPDX file and verify if component names are in current SBOM for given project-version + +optional arguments: + -h, --help show this help message and exit + --base-url BASE_URL Hub server URL e.g. https://your.blackduck.url + --token-file TOKEN_FILE + Access token file + --spdx-file SPDX_FILE + SPDX input file + --out-file OUT_FILE Unmatched components file + --project PROJECT_NAME + Project that contains the BOM components + --version VERSION_NAME + Version that contains the BOM components + --no-verify Disable TLS certificate verification + +''' + +from blackduck import Client +import argparse +import sys +import logging +import time +import json +from pprint import pprint +from spdx_tools.spdx.model.document import Document +from spdx_tools.spdx.parser.error import SPDXParsingError +from spdx_tools.spdx.parser.parse_anything import parse_file + +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") +parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", dest='token_file', required=True,help="Access token file") +parser.add_argument("--spdx-file", dest='spdx_file', required=True, help="SPDX input file") +parser.add_argument("--out-file", dest='out_file', required=True, help="Unmatched components file") +parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") +parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") +parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") +args = parser.parse_args() + +# Parse SPDX file. This can take a very long time, so do this first. +# Returns a Document object on success, otherwise raises an SPDXParsingError +try: + print("Reading SPDX file...") + start = time.process_time() + document: Document = parse_file(args.spdx_file) + print(f"SPDX parsing took {time.process_time() - start} seconds") +except SPDXParsingError: + logging.exception("Failed to parse spdx file") + sys.exit(1) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) + +# Open unmatched component file +# Will save name, spdxid, version, and origin/purl (if available) like so: +# "name": "react-bootstrap", +# "spdx_id": "SPDXRef-Pkg-react-bootstrap-2.1.2-30223", +# "version": "2.1.2", +# "origin": null +try: outfile = open(args.out_file, 'w') +except: + logging.exception("Failed to open file for writing: " + args.out_file) + sys.exit(1) + +# Saved component data to write to file +comps_out = [] + +# Fetch Project (can only have 1) +params = { + 'q': [f"name:{args.project_name}"] +} +projects = [p for p in bd.get_resource('projects', params=params) + if p['name'] == args.project_name] +assert len(projects) == 1, \ + f"There should one project named {args.project_name}. Found {len(projects)}" +project = projects[0] + +# Fetch Version (can only have 1) +params = { + 'q': [f"versionName:{args.version_name}"] +} +versions = [v for v in bd.get_resource('versions', project, params=params) + if v['versionName'] == args.version_name] +assert len(versions) == 1, \ + f"There should be 1 version named {args.version_name}. Found {len(versions)}" +version = versions[0] + +logging.debug(f"Found {project['name']}:{version['versionName']}") + +# Can now access attributes from the parsed document +# Note: The SPDX module renames tags slightly from the original json format. +#print(f"Parsed document name: {document.creation_info.name}") +#creators_as_str = ", ".join([creator.to_serialized_string() for creator in document.creation_info.creators]) +#print(f"Created on {document.creation_info.created} by {creators_as_str}") + +# A test of multiple search params.... +# this works, but it's an OR, not AND +# - we'll find everything with name OR version match +#name = "micromatch" +#ver = "4.0.2" +#params = { +# 'q': [f"componentOrVersionName:{name}"], +# 'q': [f"componentOrVersionName:{ver}"], +#} +#comps = bd.get_resource('components', version, params=params) +#for comp in comps: +# print(comp['componentName']) + +# A test of custom comp search params.... +#name = "swrightcomp" +#params = { +# 'q': [f"componentOrVersionName:{name}"], +#} +#comps = bd.get_resource('components', version, params=params) +#for comp in comps: +# print(comp['componentName']) +#quit() +# Some fun stats to track as we go +matches = 0 +nopurl = 0 +nomatch = 0 + +# Walk through each component in the SPDX file +for package in document.packages: + # spdx-tools module says only name, spdx_id, download_location are required + # We hope we'll have an external reference (pURL), but we might not. + extref = None + + # NOTE: BD can mangle the original component name + # EX: "React" -> "React from Facebook" + if package.external_references: + print(" Found external reference: " + package.external_references[0].locator) + extref = package.external_references[0].locator + + # TODO lookup KB api here + #/api/search/kb-components?filter=pURL: + + # TODO: if lookup successful, next in loop + # if : + # continue + else: + nopurl += 1 + print(" No pURL found for component: ") + print(" " + package.name) + print(" " + package.spdx_id) + print(" " + package.version) + + # Lookup existing SBOM for a match (just on name to start) + # This is a fuzzy match (see "react" for an example) + params = { + 'q': [f"componentOrVersionName:{package.name}"] + } + + # Search BOM for specific component name + comps = bd.get_resource('components', version, params=params) + # TODO investigate searching tag here + have_match = False + num_match = 0 + for comp in comps: + #pprint(bd.list_resources(comp)) + #pprint(comp) + # Check component name + version name + if comp['componentVersionName'] == package.version: + have_match = True + num_match += 1 + # TODO need to worry about multiple matches? + break + + if have_match: + matches += 1 + print("Found comp match in BOM: " + package.name) + else: + # TODO: + # 1) check if in custom component list (system-wide) + # 2) add if not there + # 3) add to project BOM + nomatch += 1 + print("May need to add this custom comp: " + package.name) + comp_data = { + "name": package.name, + "spdx_id": package.spdx_id, + "version": package.version, + "origin": extref + } + comps_out.append(comp_data) + +# Save unmatched components +json.dump(comps_out, outfile) +outfile.close() + +print("Stats: ") +print(f" Non matches: {nomatch}") +print(f" Matches: {matches}") +print(f" Packages missing purl: {nopurl}") + +# Parsed SPDX package data looks like +# Package(spdx_id='SPDXRef-Pkg-micromatch-4.0.2-30343', +# name='micromatch', +# download_location=NOASSERTION, +# version='4.0.2', +# file_name=None, +# supplier=None, +# originator=None, +# files_analyzed=True, +# verification_code=PackageVerificationCode(value='600ce1a1b891b48a20a3d395e4714f854dc6ced4', +# excluded_files=[]), +# checksums=[], +# homepage='https://www.npmjs.com/package/micromatch', +# source_info=None, +# license_concluded=LicenseSymbol('MIT', +# is_exception=False), +# license_info_from_files=[LicenseSymbol('Apache-2.0', +# is_exception=False), +# LicenseSymbol('BSD-2-Clause', +# is_exception=False), +# LicenseSymbol('ISC', +# is_exception=False), +# LicenseSymbol('JSON', +# is_exception=False), +# LicenseSymbol('LicenseRef-Historical-Permission-Notice-and-Disclaimer---sell-variant', +# is_exception=False), +# LicenseSymbol('LicenseRef-MIT-Open-Group-variant', +# is_exception=False)], +# license_declared=LicenseSymbol('MIT', +# is_exception=False), +# license_comment=None, +# copyright_text=NOASSERTION, +# summary=None, +# description=None, +# comment=None, +# external_references=[ExternalPackageRef(category=, +# reference_type='purl', +# locator='pkg:npm/micromatch@4.0.2', +# comment=None)], +# attribution_texts=[], +# primary_package_purpose=None, +# release_date=None, +# built_date=None, +# valid_until_date=None) From d9e788c8a045dac2d32a24f7a90df1839cfb08a3 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Thu, 17 Aug 2023 16:45:22 -0400 Subject: [PATCH 013/146] - Start making more modular - added find_comp_in_bom - Add KB lookup using new API + improve comp name matching - Work on SPDX validation in addition to simple parsing - Misc cleanup - Still a WIP --- examples/client/parse_spdx.py | 148 ++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 60 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 9e16f98c..6643913d 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -49,6 +49,7 @@ json pprint spdx_tools + re - Blackduck instance - API token with sufficient privileges to perform project version phase @@ -84,11 +85,37 @@ import logging import time import json +import re from pprint import pprint from spdx_tools.spdx.model.document import Document +from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.parse_anything import parse_file +# Locate component name + version in BOM +# Returns True on success, False on failure +def find_comp_in_bom(bd, compname, compver, projver): + have_match = False + num_match = 0 + + # Lookup existing SBOM for a match (just on name to start) + # This is a fuzzy match (see "react" for an example) + params = { + 'q': [f"componentOrVersionName:{compname}"] + } + + # Search BOM for specific component name + comps = bd.get_resource('components', projver, params=params) + for comp in comps: + if comp['componentName'] != compname: + # The BD API search is inexact. Force our match to be precise. + print(f"fuzzy match failed us: {comp['componentName']} vs {compname}") + continue + # Check component name + version name + if comp['componentVersionName'] == compver: + return True + return False + logging.basicConfig( level=logging.INFO, format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" @@ -115,6 +142,23 @@ logging.exception("Failed to parse spdx file") sys.exit(1) +# TODO also validate the file, which is an extra step once you have a document? +print("Validating SPDX file...") +start = time.process_time() +validation_messages = validate_full_spdx_document(document) +print(f"SPDX validation took {time.process_time() - start} seconds") +fatal = False +for validation_message in validation_messages: + if re.match(r'.*WARNING.*', validation_message.validation_message): + logging.warning(validation_message.validation_message) + if re.match(r'.*ERROR.*', validation_message.validation_message): + logging.error(validation_message.validation_message) + fatal = True + +if fatal: + print("we are dead") +quit() + with open(args.token_file, 'r') as tf: access_token = tf.readline().strip() @@ -158,93 +202,73 @@ # Can now access attributes from the parsed document # Note: The SPDX module renames tags slightly from the original json format. -#print(f"Parsed document name: {document.creation_info.name}") -#creators_as_str = ", ".join([creator.to_serialized_string() for creator in document.creation_info.creators]) -#print(f"Created on {document.creation_info.created} by {creators_as_str}") - -# A test of multiple search params.... -# this works, but it's an OR, not AND -# - we'll find everything with name OR version match -#name = "micromatch" -#ver = "4.0.2" -#params = { -# 'q': [f"componentOrVersionName:{name}"], -# 'q': [f"componentOrVersionName:{ver}"], -#} -#comps = bd.get_resource('components', version, params=params) -#for comp in comps: -# print(comp['componentName']) - -# A test of custom comp search params.... -#name = "swrightcomp" -#params = { -# 'q': [f"componentOrVersionName:{name}"], -#} -#comps = bd.get_resource('components', version, params=params) -#for comp in comps: -# print(comp['componentName']) -#quit() -# Some fun stats to track as we go + matches = 0 nopurl = 0 nomatch = 0 +# situations to consider + actions +# 1) No purl available : check SBOM for comp+ver, then add cust comp + add to SBOM +# 2) Have purl + found in KB +# - In SBOM? -> done +# - Else -> add known KB comp to SBOM +# *** this shouldn't happen in theory +# 3) Have purl + not in KB (main case we are concerned with) +# - In SBOM? (maybe already added or whatever?) -> done +# - Else -> add cust comp + add to SBOM (same as 1) + # Walk through each component in the SPDX file +package_count = 0 +packages = {} for package in document.packages: + package_count += 1 # spdx-tools module says only name, spdx_id, download_location are required # We hope we'll have an external reference (pURL), but we might not. extref = None + purlmatch = False + matchname = package.name + matchver = package.version + packages[package.name+package.version] = packages.get(package.name+package.version, 0) + 1 + #blah['zzz'] = blah.get('zzz', 0) + 1 # NOTE: BD can mangle the original component name # EX: "React" -> "React from Facebook" if package.external_references: - print(" Found external reference: " + package.external_references[0].locator) extref = package.external_references[0].locator - # TODO lookup KB api here - #/api/search/kb-components?filter=pURL: - - # TODO: if lookup successful, next in loop - # if : - # continue + # KB lookup to check for pURL match + params = { + 'packageUrl': extref + } + for result in bd.get_items("/api/search/purl-components", params=params): + # do we need to worry about more than 1 match? + print(f"Found KB match for {extref}") + purlmatch = True + #pprint(result) + # in this event, override the spdx name and use the known KB name + # (any concern for version mangling??) + if matchname != result['componentName']: + print(f"updating {matchname} -> {result['componentName']}") + matchname = result['componentName'] + # Any match means we should already have it + # But we will also check to see if the comp is in the BOM i guess else: nopurl += 1 - print(" No pURL found for component: ") + print("No pURL found for component: ") print(" " + package.name) print(" " + package.spdx_id) print(" " + package.version) - # Lookup existing SBOM for a match (just on name to start) - # This is a fuzzy match (see "react" for an example) - params = { - 'q': [f"componentOrVersionName:{package.name}"] - } - - # Search BOM for specific component name - comps = bd.get_resource('components', version, params=params) - # TODO investigate searching tag here - have_match = False - num_match = 0 - for comp in comps: - #pprint(bd.list_resources(comp)) - #pprint(comp) - # Check component name + version name - if comp['componentVersionName'] == package.version: - have_match = True - num_match += 1 - # TODO need to worry about multiple matches? - break - - if have_match: + if find_comp_in_bom(bd, matchname, matchver, version): matches += 1 - print("Found comp match in BOM: " + package.name) + print(" Found comp match in BOM: " + matchname + matchver) else: # TODO: # 1) check if in custom component list (system-wide) # 2) add if not there # 3) add to project BOM nomatch += 1 - print("May need to add this custom comp: " + package.name) + print(" Need to add custom comp: " + package.name) comp_data = { "name": package.name, "spdx_id": package.spdx_id, @@ -257,11 +281,15 @@ json.dump(comps_out, outfile) outfile.close() -print("Stats: ") +print("\nStats: ") +print("------") +print(f" SPDX packages processed: {package_count}") print(f" Non matches: {nomatch}") print(f" Matches: {matches}") print(f" Packages missing purl: {nopurl}") +pprint(packages) +print(f" {len(packages)} unique packages processed") # Parsed SPDX package data looks like # Package(spdx_id='SPDXRef-Pkg-micromatch-4.0.2-30343', # name='micromatch', From 4392280dd926c656ed99f0d17fda9d8875e5fb94 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Thu, 17 Aug 2023 16:59:19 -0400 Subject: [PATCH 014/146] just print warnings from validation messages --- examples/client/parse_spdx.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 6643913d..ab7a122f 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -147,17 +147,12 @@ def find_comp_in_bom(bd, compname, compver, projver): start = time.process_time() validation_messages = validate_full_spdx_document(document) print(f"SPDX validation took {time.process_time() - start} seconds") -fatal = False + +# TODO is there a way to distinguish between something fatal and something +# BD can deal with? +# I guess we can just print all the msgs and then also exit when the import fails.. for validation_message in validation_messages: - if re.match(r'.*WARNING.*', validation_message.validation_message): - logging.warning(validation_message.validation_message) - if re.match(r'.*ERROR.*', validation_message.validation_message): - logging.error(validation_message.validation_message) - fatal = True - -if fatal: - print("we are dead") -quit() + logging.warning(validation_message.validation_message) with open(args.token_file, 'r') as tf: access_token = tf.readline().strip() From 95079694fd19d255a5f0f688575eefbfed23d27b Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Fri, 18 Aug 2023 12:18:11 -0400 Subject: [PATCH 015/146] signficant refactoring, plus adding custom component lookup and add functionality. WIP. --- examples/client/parse_spdx.py | 291 +++++++++++++++++++++------------- 1 file changed, 182 insertions(+), 109 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index ab7a122f..eeba17e1 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -92,6 +92,59 @@ from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.parse_anything import parse_file + + # Returns SPDX Document object on success, otherwise exits on parse failure +def spdx_parse(file): + print("Parsing SPDX file...") + start = time.process_time() + try: + document: Document = parse_file(file) + print(f"SPDX parsing took {time.process_time() - start} seconds") + return(document) + except SPDXParsingError: + logging.exception("Failed to parse spdx file") + sys.exit(1) + +# Validates the SPDX file. Logs all validation messages as warnings. +def spdx_validate(document): + print("Validating SPDX file...") + start = time.process_time() + validation_messages = validate_full_spdx_document(document) + print(f"SPDX validation took {time.process_time() - start} seconds") + + # TODO is there a way to distinguish between something fatal and something + # BD can deal with? + # TODO - this can take forever, so add an optional --skip-validation flag + for validation_message in validation_messages: + # Just printing these messages intead of exiting. Later when we try to import + # the file to BD, let's plan to exit if it fails. Seeing lots of errors in the + # sample data. + logging.warning(validation_message.validation_message) + +# Lookup the given matchname in the KB +# Logs a successful match +# Return the boolean purlmatch and matchname, which we might change from +# its original value -- we will force it to be the same as the name in BD. +# That way we can more accurately search the BOM later. +def find_comp_in_kb(matchname, extref): + # KB lookup to check for pURL match + purlmatch = False + params = { + 'packageUrl': extref + } + # TODO any other action to take here? + # We should probably track KB matches? + for result in bd.get_items("/api/search/purl-components", params=params): + # do we need to worry about more than 1 match? + #print(f"Found KB match for {extref}") + purlmatch = True + # in this event, override the spdx name and use the known KB name + # (is version mangling possible??) + if matchname != result['componentName']: + print(f"updating {matchname} -> {result['componentName']}") + return(purlmatch, result['componentName']) + return(purlmatch, matchname) + # Locate component name + version in BOM # Returns True on success, False on failure def find_comp_in_bom(bd, compname, compver, projver): @@ -109,17 +162,89 @@ def find_comp_in_bom(bd, compname, compver, projver): for comp in comps: if comp['componentName'] != compname: # The BD API search is inexact. Force our match to be precise. - print(f"fuzzy match failed us: {comp['componentName']} vs {compname}") + #print(f"fuzzy match failed us: {comp['componentName']} vs {compname}") continue # Check component name + version name if comp['componentVersionName'] == compver: return True return False -logging.basicConfig( - level=logging.INFO, - format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" -) + +# Returns: +# CompMatch - Contains matched component object, None for no match +# FoundVer - Boolen: True if matched the custom component version +def find_cust_comp(cust_comp_name, cust_comp_version): + params = { + 'q': [f"name:{cust_comp_name}"] + } + + matched_comp = None + # Relies on internal header + headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} + ver_match = False + for comp in bd.get_resource('components', params=params, headers=headers): + print(f"{comp['name']}") + if cust_comp_name != comp['name']: + # Skip it. We want to be precise in our matching, despite the API. + continue + matched_comp = comp + # Check version + for version in bd.get_resource('versions', comp): + if cust_comp_version == version['versionName']: + # Successfully matched both name and version + ver_match = True + return(matched_comp, ver_match) + + return(matched_comp, ver_match) + + +# Returns URL of matching license +# Exits on failure, we assume it must exist - TODO could probably just create this? +def get_license_url(license_name): + params = { + 'q': [f"name:{license_name}"] + } + for result in bd.get_items("/api/licenses", params=params): + # Added precise matching in case of a situation like "NOASSERTION" & "NOASSERTION2" + if (result['name'] == license_name): + return(result['_meta']['href']) + + logging.error(f"Failed to find license {license_name}") + sys.exit(1) + +def create_cust_comp(name, version, license, approval): + print(f"Adding custom component: {name} {version}") + license_url = get_license_url(license) + data = { + 'name': name, + 'version' : { + 'versionName' : version, + 'license' : { + 'license' : license_url + }, + }, + 'approvalStatus': approval + } + response = bd.session.post("api/components", json=data) + pprint(response) + + # TODO validate response + # looks like a 412 if it already existed + +# Create a version for a custom component that already exists +# The comp argument is the component object from previous lookup +def create_cust_comp_ver(comp, version, license): + print(f"Adding version {version} to custom component {comp['name']}") + license_url = get_license_url(license) + data = { + 'versionName' : version, + 'license' : { + 'license' : license_url + }, + } + response = bd.session.post(comp['_meta']['href'] + "/versions", json=data) + pprint(response) + # TODO validate response parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") @@ -129,30 +254,17 @@ def find_comp_in_bom(bd, compname, compver, projver): parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") +parser.add_argument("--no-spdx-validate", dest='spdx_validate', action='store_false', help="Disable SPDX validation") args = parser.parse_args() -# Parse SPDX file. This can take a very long time, so do this first. -# Returns a Document object on success, otherwise raises an SPDXParsingError -try: - print("Reading SPDX file...") - start = time.process_time() - document: Document = parse_file(args.spdx_file) - print(f"SPDX parsing took {time.process_time() - start} seconds") -except SPDXParsingError: - logging.exception("Failed to parse spdx file") - sys.exit(1) - -# TODO also validate the file, which is an extra step once you have a document? -print("Validating SPDX file...") -start = time.process_time() -validation_messages = validate_full_spdx_document(document) -print(f"SPDX validation took {time.process_time() - start} seconds") +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) -# TODO is there a way to distinguish between something fatal and something -# BD can deal with? -# I guess we can just print all the msgs and then also exit when the import fails.. -for validation_message in validation_messages: - logging.warning(validation_message.validation_message) +document = spdx_parse(args.spdx_file) +if (args.spdx_validate): + spdx_validate(document) with open(args.token_file, 'r') as tf: access_token = tf.readline().strip() @@ -160,7 +272,7 @@ def find_comp_in_bom(bd, compname, compver, projver): bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) # Open unmatched component file -# Will save name, spdxid, version, and origin/purl (if available) like so: +# Will save name, spdxid, version, and origin/purl for later in json format: # "name": "react-bootstrap", # "spdx_id": "SPDXRef-Pkg-react-bootstrap-2.1.2-30223", # "version": "2.1.2", @@ -195,12 +307,6 @@ def find_comp_in_bom(bd, compname, compver, projver): logging.debug(f"Found {project['name']}:{version['versionName']}") -# Can now access attributes from the parsed document -# Note: The SPDX module renames tags slightly from the original json format. - -matches = 0 -nopurl = 0 -nomatch = 0 # situations to consider + actions # 1) No purl available : check SBOM for comp+ver, then add cust comp + add to SBOM @@ -212,41 +318,33 @@ def find_comp_in_bom(bd, compname, compver, projver): # - In SBOM? (maybe already added or whatever?) -> done # - Else -> add cust comp + add to SBOM (same as 1) -# Walk through each component in the SPDX file +# Stats to track +bom_matches = 0 +kb_matches = 0 +nopurl = 0 +nomatch = 0 package_count = 0 +cust_comp_count = 0 +cust_ver_count = 0 +# Saving all encountered components by their name+version (watching for repeats) packages = {} + +# Walk through each component in the SPDX file for package in document.packages: package_count += 1 - # spdx-tools module says only name, spdx_id, download_location are required # We hope we'll have an external reference (pURL), but we might not. extref = None purlmatch = False matchname = package.name matchver = package.version - packages[package.name+package.version] = packages.get(package.name+package.version, 0) + 1 - #blah['zzz'] = blah.get('zzz', 0) + 1 + # Tracking unique package name + version from spdx file + packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 - # NOTE: BD can mangle the original component name + # NOTE: BD can change the original component name # EX: "React" -> "React from Facebook" if package.external_references: - extref = package.external_references[0].locator - - # KB lookup to check for pURL match - params = { - 'packageUrl': extref - } - for result in bd.get_items("/api/search/purl-components", params=params): - # do we need to worry about more than 1 match? - print(f"Found KB match for {extref}") - purlmatch = True - #pprint(result) - # in this event, override the spdx name and use the known KB name - # (any concern for version mangling??) - if matchname != result['componentName']: - print(f"updating {matchname} -> {result['componentName']}") - matchname = result['componentName'] - # Any match means we should already have it - # But we will also check to see if the comp is in the BOM i guess + inkb, matchname = find_comp_in_kb(matchname, package.external_references[0].locator) + if inkb: kb_matches += 1 else: nopurl += 1 print("No pURL found for component: ") @@ -255,15 +353,10 @@ def find_comp_in_bom(bd, compname, compver, projver): print(" " + package.version) if find_comp_in_bom(bd, matchname, matchver, version): - matches += 1 - print(" Found comp match in BOM: " + matchname + matchver) + bom_matches += 1 + #print(" Found comp match in BOM: " + matchname + matchver) else: - # TODO: - # 1) check if in custom component list (system-wide) - # 2) add if not there - # 3) add to project BOM nomatch += 1 - print(" Need to add custom comp: " + package.name) comp_data = { "name": package.name, "spdx_id": package.spdx_id, @@ -271,6 +364,27 @@ def find_comp_in_bom(bd, compname, compver, projver): "origin": extref } comps_out.append(comp_data) + + # Check if custom component already exists + comp_match, found_ver = find_cust_comp(package.name, package.version) + + # TODO make these optional args with defaults + license = "NOASSERTION" + approval = "UNREVIEWED" + if not comp_match: + cust_comp_count += 1 + create_cust_comp(package.name, package.version, license, approval) + elif comp_match and not found_ver: + cust_ver_count += 1 + print("Adding custom component version...") + create_cust_comp_ver(comp_match, package.version, license) + else: + # nothing to do? + print("probably found name and ver") + + # TODO write sbom add code + #add_to_sbom() + # Save unmatched components json.dump(comps_out, outfile) @@ -280,52 +394,11 @@ def find_comp_in_bom(bd, compname, compver, projver): print("------") print(f" SPDX packages processed: {package_count}") print(f" Non matches: {nomatch}") -print(f" Matches: {matches}") +print(f" KB matches: {kb_matches}") +print(f" BOM matches: {bom_matches}") print(f" Packages missing purl: {nopurl}") +print(f" Custom components created: {cust_comp_count}") +print(f" Custom component versions created: {cust_ver_count}") -pprint(packages) +#pprint(packages) print(f" {len(packages)} unique packages processed") -# Parsed SPDX package data looks like -# Package(spdx_id='SPDXRef-Pkg-micromatch-4.0.2-30343', -# name='micromatch', -# download_location=NOASSERTION, -# version='4.0.2', -# file_name=None, -# supplier=None, -# originator=None, -# files_analyzed=True, -# verification_code=PackageVerificationCode(value='600ce1a1b891b48a20a3d395e4714f854dc6ced4', -# excluded_files=[]), -# checksums=[], -# homepage='https://www.npmjs.com/package/micromatch', -# source_info=None, -# license_concluded=LicenseSymbol('MIT', -# is_exception=False), -# license_info_from_files=[LicenseSymbol('Apache-2.0', -# is_exception=False), -# LicenseSymbol('BSD-2-Clause', -# is_exception=False), -# LicenseSymbol('ISC', -# is_exception=False), -# LicenseSymbol('JSON', -# is_exception=False), -# LicenseSymbol('LicenseRef-Historical-Permission-Notice-and-Disclaimer---sell-variant', -# is_exception=False), -# LicenseSymbol('LicenseRef-MIT-Open-Group-variant', -# is_exception=False)], -# license_declared=LicenseSymbol('MIT', -# is_exception=False), -# license_comment=None, -# copyright_text=NOASSERTION, -# summary=None, -# description=None, -# comment=None, -# external_references=[ExternalPackageRef(category=, -# reference_type='purl', -# locator='pkg:npm/micromatch@4.0.2', -# comment=None)], -# attribution_texts=[], -# primary_package_purpose=None, -# release_date=None, -# built_date=None, -# valid_until_date=None) From d13ae2492d7a7296946b9ad76ec8e4d6e3fe249d Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Fri, 18 Aug 2023 16:02:49 -0400 Subject: [PATCH 016/146] tons of cleanup/bugfixes + add_to_sbom utility --- examples/client/parse_spdx.py | 93 ++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index eeba17e1..3407e5ed 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -124,7 +124,7 @@ def spdx_validate(document): # Lookup the given matchname in the KB # Logs a successful match # Return the boolean purlmatch and matchname, which we might change from -# its original value -- we will force it to be the same as the name in BD. +# its original value -- we will force it to be the same as the name in the KB # That way we can more accurately search the BOM later. def find_comp_in_kb(matchname, extref): # KB lookup to check for pURL match @@ -135,13 +135,12 @@ def find_comp_in_kb(matchname, extref): # TODO any other action to take here? # We should probably track KB matches? for result in bd.get_items("/api/search/purl-components", params=params): - # do we need to worry about more than 1 match? - #print(f"Found KB match for {extref}") + # TODO do we need to worry about more than 1 match? purlmatch = True # in this event, override the spdx name and use the known KB name - # (is version mangling possible??) + # TODO: is version mangling possible? if matchname != result['componentName']: - print(f"updating {matchname} -> {result['componentName']}") + print(f"Renaming {matchname} -> {result['componentName']}") return(purlmatch, result['componentName']) return(purlmatch, matchname) @@ -151,7 +150,7 @@ def find_comp_in_bom(bd, compname, compver, projver): have_match = False num_match = 0 - # Lookup existing SBOM for a match (just on name to start) + # Lookup existing SBOM for a match # This is a fuzzy match (see "react" for an example) params = { 'q': [f"componentOrVersionName:{compname}"] @@ -162,44 +161,47 @@ def find_comp_in_bom(bd, compname, compver, projver): for comp in comps: if comp['componentName'] != compname: # The BD API search is inexact. Force our match to be precise. - #print(f"fuzzy match failed us: {comp['componentName']} vs {compname}") continue # Check component name + version name - if comp['componentVersionName'] == compver: - return True + try: + if comp['componentVersionName'] == compver: + return True + except: + # Handle situation where it's missing the version name for some reason + print(f"comp {compname} in BOM has no version!") + return False return False # Returns: -# CompMatch - Contains matched component object, None for no match -# FoundVer - Boolen: True if matched the custom component version +# CompMatch - Contains matched component url, None for no match +# VerMatch - Contains matched component verison url, None for no match def find_cust_comp(cust_comp_name, cust_comp_version): params = { 'q': [f"name:{cust_comp_name}"] } matched_comp = None + matched_ver = None # Relies on internal header headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} - ver_match = False for comp in bd.get_resource('components', params=params, headers=headers): - print(f"{comp['name']}") if cust_comp_name != comp['name']: # Skip it. We want to be precise in our matching, despite the API. continue - matched_comp = comp + matched_comp = comp['_meta']['href'] # Check version for version in bd.get_resource('versions', comp): if cust_comp_version == version['versionName']: # Successfully matched both name and version - ver_match = True - return(matched_comp, ver_match) + matched_ver = version['_meta']['href'] + return(matched_comp, matched_ver) - return(matched_comp, ver_match) + return(matched_comp, matched_ver) # Returns URL of matching license -# Exits on failure, we assume it must exist - TODO could probably just create this? +# Exits on failure, we assume it must pre-exist - TODO could probably just create this? def get_license_url(license_name): params = { 'q': [f"name:{license_name}"] @@ -212,6 +214,7 @@ def get_license_url(license_name): logging.error(f"Failed to find license {license_name}") sys.exit(1) +# Returns the URL for the newly created component version URL if successful def create_cust_comp(name, version, license, approval): print(f"Adding custom component: {name} {version}") license_url = get_license_url(license) @@ -225,16 +228,19 @@ def create_cust_comp(name, version, license, approval): }, 'approvalStatus': approval } - response = bd.session.post("api/components", json=data) - pprint(response) - # TODO validate response # looks like a 412 if it already existed + response = bd.session.post("api/components", json=data) + # should be guaranteed 1 version because we just created it! + # TODO put in a fail-safe + for version in bd.get_items(response.links['versions']['url']): + return(version['_meta']['href']) + + #return(response.links['self']['url']) # Create a version for a custom component that already exists -# The comp argument is the component object from previous lookup -def create_cust_comp_ver(comp, version, license): - print(f"Adding version {version} to custom component {comp['name']}") +# Returns the component version url just created +def create_cust_comp_ver(comp_url, version, license): license_url = get_license_url(license) data = { 'versionName' : version, @@ -242,9 +248,18 @@ def create_cust_comp_ver(comp, version, license): 'license' : license_url }, } - response = bd.session.post(comp['_meta']['href'] + "/versions", json=data) - pprint(response) + #response = bd.session.post(comp['_meta']['href'] + "/versions", json=data) + response = bd.session.post(comp_url + "/versions", json=data) # TODO validate response + return(response.links['versions']['url']) + +# Add specified component version url to our project+version SBOM +def add_to_sbom(proj_version_url, comp_ver_url): + data = { + 'component': comp_ver_url + } + # TODO validate response + response = bd.session.post(proj_version_url + "/components", json=data) parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") @@ -294,7 +309,6 @@ def create_cust_comp_ver(comp, version, license): assert len(projects) == 1, \ f"There should one project named {args.project_name}. Found {len(projects)}" project = projects[0] - # Fetch Version (can only have 1) params = { 'q': [f"versionName:{args.version_name}"] @@ -304,6 +318,7 @@ def create_cust_comp_ver(comp, version, license): assert len(versions) == 1, \ f"There should be 1 version named {args.version_name}. Found {len(versions)}" version = versions[0] +proj_version_url = version['_meta']['href'] logging.debug(f"Found {project['name']}:{version['versionName']}") @@ -366,26 +381,27 @@ def create_cust_comp_ver(comp, version, license): comps_out.append(comp_data) # Check if custom component already exists - comp_match, found_ver = find_cust_comp(package.name, package.version) + comp_url, comp_ver_url = find_cust_comp(package.name, package.version) # TODO make these optional args with defaults license = "NOASSERTION" approval = "UNREVIEWED" - if not comp_match: + if not comp_url: cust_comp_count += 1 - create_cust_comp(package.name, package.version, license, approval) - elif comp_match and not found_ver: + comp_ver_url = create_cust_comp(package.name, package.version, + license, approval) + elif comp_url and not comp_ver_url: cust_ver_count += 1 - print("Adding custom component version...") - create_cust_comp_ver(comp_match, package.version, license) + print(f"Adding version {package.version} to custom component {package.name}") + comp_ver_url = create_cust_comp_ver(comp_url, package.version, license) else: - # nothing to do? - print("probably found name and ver") + print("Custom component already exists, not in SBOM") - # TODO write sbom add code - #add_to_sbom() + # is this possible? i don't think so + assert(comp_ver_url), f"No comp_ver URL found for {package.name} {package.version}" + print(f"Adding component to SBOM: {package.name} {package.version}") + add_to_sbom(proj_version_url, comp_ver_url) - # Save unmatched components json.dump(comps_out, outfile) outfile.close() @@ -399,6 +415,5 @@ def create_cust_comp_ver(comp, version, license): print(f" Packages missing purl: {nopurl}") print(f" Custom components created: {cust_comp_count}") print(f" Custom component versions created: {cust_ver_count}") - #pprint(packages) print(f" {len(packages)} unique packages processed") From d832762495a68ceabc8c8fa231d29347ab57d806 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Mon, 21 Aug 2023 15:04:54 -0400 Subject: [PATCH 017/146] misc cleanup/bugfixes. make --license an optional arg that defaults to NOASSERTION --- examples/client/parse_spdx.py | 106 ++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 23 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 3407e5ed..aecc0468 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -50,6 +50,7 @@ pprint spdx_tools re + pathlib - Blackduck instance - API token with sufficient privileges to perform project version phase @@ -87,13 +88,16 @@ import json import re from pprint import pprint +from pathlib import Path from spdx_tools.spdx.model.document import Document from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.parse_anything import parse_file - - # Returns SPDX Document object on success, otherwise exits on parse failure +# TODO what happens if file doesn't exist? +# Returns SPDX Document object on success, otherwise exits on parse failure +# Input: file = Filename to process +# Returns: SPDX document object def spdx_parse(file): print("Parsing SPDX file...") start = time.process_time() @@ -121,6 +125,7 @@ def spdx_validate(document): # sample data. logging.warning(validation_message.validation_message) +# TODO is it possible to make this a case-insensitive match? # Lookup the given matchname in the KB # Logs a successful match # Return the boolean purlmatch and matchname, which we might change from @@ -144,6 +149,7 @@ def find_comp_in_kb(matchname, extref): return(purlmatch, result['componentName']) return(purlmatch, matchname) +# TODO is it possible to make this a case-insensitive match? # Locate component name + version in BOM # Returns True on success, False on failure def find_comp_in_bom(bd, compname, compver, projver): @@ -173,6 +179,7 @@ def find_comp_in_bom(bd, compname, compver, projver): return False +# TODO is it possible to make this a case-insensitive match? # Returns: # CompMatch - Contains matched component url, None for no match # VerMatch - Contains matched component verison url, None for no match @@ -202,6 +209,7 @@ def find_cust_comp(cust_comp_name, cust_comp_version): # Returns URL of matching license # Exits on failure, we assume it must pre-exist - TODO could probably just create this? +# Note: License name search is case-sensitive def get_license_url(license_name): params = { 'q': [f"name:{license_name}"] @@ -214,8 +222,13 @@ def get_license_url(license_name): logging.error(f"Failed to find license {license_name}") sys.exit(1) +# Create a custom component +# Inputs: +# name - Name of component to add +# version - Version of component to add +# license - License name # Returns the URL for the newly created component version URL if successful -def create_cust_comp(name, version, license, approval): +def create_cust_comp(name, version, license): print(f"Adding custom component: {name} {version}") license_url = get_license_url(license) data = { @@ -225,21 +238,34 @@ def create_cust_comp(name, version, license, approval): 'license' : { 'license' : license_url }, - }, - 'approvalStatus': approval + } } - # TODO validate response - # looks like a 412 if it already existed response = bd.session.post("api/components", json=data) - # should be guaranteed 1 version because we just created it! - # TODO put in a fail-safe + logging.debug(response) + if response.status_code == 412: + # Shouldn't be possible. We checked for existence earlier. + logging.error(f"Component {name} already exists") + sys.exit(1) + + if response.status_code != 201: + # Shouldn't be possible. We checked for existence earlier. + logging.error(response.json()['errors'][0]['errorMessage']) + logging.error(f"Status code {response.status_code}") + sys.exit(1) + + # Should be guaranteed 1 version because we just created it! for version in bd.get_items(response.links['versions']['url']): return(version['_meta']['href']) - #return(response.links['self']['url']) # Create a version for a custom component that already exists -# Returns the component version url just created +# +# Inputs: +# comp_url - API URL of the component to update +# version - Version to add to existing component +# license - License to use for version +# +# Returns: component version url just created def create_cust_comp_ver(comp_url, version, license): license_url = get_license_url(license) data = { @@ -248,18 +274,33 @@ def create_cust_comp_ver(comp_url, version, license): 'license' : license_url }, } - #response = bd.session.post(comp['_meta']['href'] + "/versions", json=data) response = bd.session.post(comp_url + "/versions", json=data) - # TODO validate response - return(response.links['versions']['url']) + logging.debug(response) + if response.status_code == 412: + # Shouldn't be possible. We checked for existence earlier. + logging.error(f"Version {version} already exists for component") + sys.exit(1) + + # necessary? + if response.status_code != 201: + logging.error(f"Failed to add Version {version} to component") + sys.exit(1) + + return(response.links['self']['url']) # Add specified component version url to our project+version SBOM +# Inputs: +# proj_version_url: API URL for a project+version to update +# comp_ver_url: API URL of a component+version to add def add_to_sbom(proj_version_url, comp_ver_url): data = { 'component': comp_ver_url } - # TODO validate response response = bd.session.post(proj_version_url + "/components", json=data) + if (response.status_code != 200): + logging.error(response.json()['errors'][0]['errorMessage']) + logging.error(f"Status code {response.status_code}") + sys.exit(1) parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") @@ -268,6 +309,7 @@ def add_to_sbom(proj_version_url, comp_ver_url): parser.add_argument("--out-file", dest='out_file', required=True, help="Unmatched components file") parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") +parser.add_argument("--license", dest='license_name', required=False, default="NOASSERTION", help="License name to use for custom components") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") parser.add_argument("--no-spdx-validate", dest='spdx_validate', action='store_false', help="Disable SPDX validation") args = parser.parse_args() @@ -277,21 +319,38 @@ def add_to_sbom(proj_version_url, comp_ver_url): format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" ) -document = spdx_parse(args.spdx_file) -if (args.spdx_validate): - spdx_validate(document) +if (Path(args.spdx_file).is_file()): + document = spdx_parse(args.spdx_file) + if (args.spdx_validate): + spdx_validate(document) +else: + logging.error(f"Invalid SPDX file: {args.spdx_file}") + sys.exit(1) with open(args.token_file, 'r') as tf: access_token = tf.readline().strip() bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) +# some little debug/test stubs +# TODO: delete these +#comp_ver_url = create_cust_comp("MY COMPONENT z", "1", args.license_name) +# +#comp_url = "https://purl-validation.saas-staging.blackduck.com/api/components/886c04d4-28ce-4a27-be4c-f083e73a9f69" +#comp_ver_url = create_cust_comp_ver(comp_url, "701", "NOASSERTION") +# +#pv = "https://purl-validation.saas-staging.blackduck.com/api/projects/14b714d0-fa37-4684-86cc-ed4e7cc64b89/versions/b8426ca3-1e27-4045-843b-003eca72f98e" +#cv = "https://purl-validation.saas-staging.blackduck.com/api/components/886c04d4-28ce-4a27-be4c-f083e73a9f69/versions/56f64b7f-c284-457d-b593-0cf19a272a19" +#add_to_sbom(pv, cv) +#quit() + # Open unmatched component file # Will save name, spdxid, version, and origin/purl for later in json format: # "name": "react-bootstrap", # "spdx_id": "SPDXRef-Pkg-react-bootstrap-2.1.2-30223", # "version": "2.1.2", # "origin": null +# TODO this try/except actually isn't right try: outfile = open(args.out_file, 'w') except: logging.exception("Failed to open file for writing: " + args.out_file) @@ -383,17 +442,18 @@ def add_to_sbom(proj_version_url, comp_ver_url): # Check if custom component already exists comp_url, comp_ver_url = find_cust_comp(package.name, package.version) - # TODO make these optional args with defaults - license = "NOASSERTION" - approval = "UNREVIEWED" if not comp_url: + # Custom component did not exist, so create it cust_comp_count += 1 comp_ver_url = create_cust_comp(package.name, package.version, - license, approval) + args.license_name, approval) elif comp_url and not comp_ver_url: + # Custom component existed, but not the version we care about cust_ver_count += 1 print(f"Adding version {package.version} to custom component {package.name}") - comp_ver_url = create_cust_comp_ver(comp_url, package.version, license) + comp_ver_url = create_cust_comp_ver(comp_url, package.version, args.license_name) + # DEBUG + quit() else: print("Custom component already exists, not in SBOM") From 77046c8a67f753920cbd8a31407e033708a9d875 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Thu, 24 Aug 2023 16:02:31 -0400 Subject: [PATCH 018/146] Add SPDX upload functionality and poll for successful scan. Tons of cleanup, assorted bugfixes, and comments. --- examples/client/parse_spdx.py | 239 ++++++++++++++++++++++++++++------ 1 file changed, 196 insertions(+), 43 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index aecc0468..06655a21 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -94,7 +94,6 @@ from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.parse_anything import parse_file -# TODO what happens if file doesn't exist? # Returns SPDX Document object on success, otherwise exits on parse failure # Input: file = Filename to process # Returns: SPDX document object @@ -110,6 +109,7 @@ def spdx_parse(file): sys.exit(1) # Validates the SPDX file. Logs all validation messages as warnings. +# Input: SPDX document object def spdx_validate(document): print("Validating SPDX file...") start = time.process_time() @@ -125,34 +125,153 @@ def spdx_validate(document): # sample data. logging.warning(validation_message.validation_message) -# TODO is it possible to make this a case-insensitive match? -# Lookup the given matchname in the KB -# Logs a successful match -# Return the boolean purlmatch and matchname, which we might change from -# its original value -- we will force it to be the same as the name in the KB -# That way we can more accurately search the BOM later. -def find_comp_in_kb(matchname, extref): - # KB lookup to check for pURL match +# Returns MIME type to provide to scan API +# Input: filename to check +def get_sbom_mime_type(filename): + with open(filename, 'r') as f: + data = f.readlines() + content = " ".join(data) + if 'CycloneDX' in content: + return 'application/vnd.cyclonedx' + if 'SPDX' in content: + return 'application/spdx' + return None + +# Poll for successful scan of SBOM. +# Input: Name of SBOM document (not the filename, the name defined inside the json body) +# Returns on success. Errors will result in fatal exit. +def poll_for_upload(sbom_name): + max_retries = 30 + sleep_time = 10 + matched_scan = False + + # Search for the latest scan matching our SBOM + # This might be a risk for a race condition + params = { + 'q': [f"name:{sbom_name}"], + 'sort': ["updatedAt: ASC"] + } + + cls = bd.get_resource('codeLocations', params=params) + for cl in cls: + # Force exact match of: spdx_doc_name + " spdx/sbom" + # BD appends the "spdx/sbom" string to the name. + if cl['name'] != sbom_name + " spdx/sbom": + continue + + matched_scan = True + for link in (cl['_meta']['links']): + # Locate the scans URL to check for status + if link['rel'] == "scans": + summaries_url = link['href'] + break + + assert(summaries_url) + params = { + 'sort': ["updatedAt: ASC"] + } + + while (max_retries): + max_retries -= 1 + for item in bd.get_items(summaries_url, params=params): + # Only checking the first item as it's the most recent + if item['scanState'] == "SUCCESS": + print("Scan complete") + return + elif item['scanState'] == "FAILURE": + logging.error(f"SPDX Scan Failure: {item['statusMessage']}") + sys.exit(1) + else: + # Only other state should be "STARTED" -- keep polling + print(f"Waiting for status success, currently: {item['scanState']}") + time.sleep(sleep_time) + # Break out of for loop so we always check the most recent + break + + # Handle various errors that might happen + if max_retries == 0: + logging.error("Failed to verify successful SPDX Scan in {max_retries * sleep_time} seconds") + elif not matched_scan: + logging.error(f"No scan found for SBOM: {sbom_name}") + else: + logging.error(f"Unable to verify successful scan of SBOM: {sbom_name}") + + sys.exit(1) + +# TODO do we care about project_groups? +# Upload provided SBOM file to Black Duck +# Inputs: +# filename - Name of file to upload +# project - Project name to map to +# version - Version name to map to +def upload_sbom_file(filename, project, version): + mime_type = get_sbom_mime_type(filename) + if not mime_type: + logging.error(f"Could not identify file content for {filename}") + sys.exit(1) + files = {"file": (filename, open(filename,"rb"), mime_type)} + fields = {"projectName": project, "versionName": version} + response = bd.session.post("/api/scan/data", files = files, data=fields) + logging.info(response) + + if response.status_code == 409: + logging.info(f"File {filename} is already mapped to a different project version") + + if response.status_code != 201: + logging.error(f"Failed to upload SPDX file:") + try: + pprint(response.json()['errorMessage']) + except: + logging.error(f"Status code {response.status_code}") + sys.exit(1) + + +# Lookup the given pURL in the BD KB. +# If successfully matched, update the associated package name and version with the data from the KB. +# This will improve the accuracy of later lookups. We are replacing the SPDX input data with the +# data stored in the KB. +# +# Inputs: +# matchname - Name of package from the SPDX input file +# matchver - Version of package +# extref - pURL to look up +# +# Returns: +# purlmatch - boolean (True if successful KB lookup) +# matchname - Original parameter OR updated to reflect KB lookup name +# matchver - Original parameter OR updated to reflect KB lookup version +def find_comp_in_kb(matchname, matchver, extref): purlmatch = False params = { 'packageUrl': extref } - # TODO any other action to take here? - # We should probably track KB matches? for result in bd.get_items("/api/search/purl-components", params=params): - # TODO do we need to worry about more than 1 match? + # This query should result in exactly 1 match purlmatch = True - # in this event, override the spdx name and use the known KB name - # TODO: is version mangling possible? + # Override the spdx name and use the known KB name if matchname != result['componentName']: print(f"Renaming {matchname} -> {result['componentName']}") - return(purlmatch, result['componentName']) - return(purlmatch, matchname) + matchname = result['componentName'] + # Override the spdx version and use the string from KB + # for example, v2.8.5 -> 2.8.5 + if matchver != result['versionName']: + print(f"Renaming {matchver} -> {result['versionName']}") + matchver = result['versionName'] + + return(purlmatch, matchname, matchver) + + # fall through -- lookup failed, so we keep the original name/ver + return(purlmatch, matchname, matchver) + -# TODO is it possible to make this a case-insensitive match? # Locate component name + version in BOM -# Returns True on success, False on failure -def find_comp_in_bom(bd, compname, compver, projver): +# Inputs: +# compname - Component name to locate +# compver - Component version to locate +# projver - Project version to locate component in BOM +# +# Returns: True on success, False on failure +def find_comp_in_bom(compname, compver, projver): have_match = False num_match = 0 @@ -179,37 +298,49 @@ def find_comp_in_bom(bd, compname, compver, projver): return False -# TODO is it possible to make this a case-insensitive match? +# Verifies if a custom component and version already exist in the system +# +# Inputs: +# compname - Component name to locate +# compver - Component version to locate # Returns: # CompMatch - Contains matched component url, None for no match # VerMatch - Contains matched component verison url, None for no match -def find_cust_comp(cust_comp_name, cust_comp_version): +def find_cust_comp(compname, compver): params = { - 'q': [f"name:{cust_comp_name}"] + 'q': [f"name:{compname}"] } matched_comp = None matched_ver = None - # Relies on internal header + # Warning: Relies on internal header headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} for comp in bd.get_resource('components', params=params, headers=headers): - if cust_comp_name != comp['name']: - # Skip it. We want to be precise in our matching, despite the API. + if compname == comp['name']: + # Force exact match + matched_comp = comp['_meta']['href'] + else: + # Keep checking search results continue - matched_comp = comp['_meta']['href'] + # Check version for version in bd.get_resource('versions', comp): - if cust_comp_version == version['versionName']: + if compver == version['versionName']: # Successfully matched both name and version matched_ver = version['_meta']['href'] return(matched_comp, matched_ver) - return(matched_comp, matched_ver) + # If we got this far, break out of the loop + # We matched the component, but not the version + break + return(matched_comp, matched_ver) -# Returns URL of matching license -# Exits on failure, we assume it must pre-exist - TODO could probably just create this? -# Note: License name search is case-sensitive +# Find URL of license to use for custom compnent creation +# Inputs: +# license_name - Name of license to locate (case-sensitive) +# +# Returns: URL of license successfully matched. Failures are fatal. def get_license_url(license_name): params = { 'q': [f"name:{license_name}"] @@ -292,6 +423,7 @@ def create_cust_comp_ver(comp_url, version, license): # Inputs: # proj_version_url: API URL for a project+version to update # comp_ver_url: API URL of a component+version to add +# Prints out any errors encountered. Errors are fatal. def add_to_sbom(proj_version_url, comp_ver_url): data = { 'component': comp_ver_url @@ -302,6 +434,7 @@ def add_to_sbom(proj_version_url, comp_ver_url): logging.error(f"Status code {response.status_code}") sys.exit(1) + parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") parser.add_argument("--token-file", dest='token_file', required=True,help="Access token file") @@ -309,7 +442,7 @@ def add_to_sbom(proj_version_url, comp_ver_url): parser.add_argument("--out-file", dest='out_file', required=True, help="Unmatched components file") parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") -parser.add_argument("--license", dest='license_name', required=False, default="NOASSERTION", help="License name to use for custom components") +parser.add_argument("--license", dest='license_name', required=False, default="NOASSERTION", help="License name to use for custom components (default: NOASSERTION)") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") parser.add_argument("--no-spdx-validate", dest='spdx_validate', action='store_false', help="Disable SPDX validation") args = parser.parse_args() @@ -324,16 +457,32 @@ def add_to_sbom(proj_version_url, comp_ver_url): if (args.spdx_validate): spdx_validate(document) else: - logging.error(f"Invalid SPDX file: {args.spdx_file}") + logging.error(f"Could not open SPDX file: {args.spdx_file}") sys.exit(1) with open(args.token_file, 'r') as tf: access_token = tf.readline().strip() +global bd bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) +#pprint(bd.list_resources()) + +upload_sbom_file(args.spdx_file, args.project_name, args.version_name) +# This will exit if it fails +poll_for_upload(document.creation_info.name) + # some little debug/test stubs # TODO: delete these +#matchcomp, matchver = find_cust_comp("ipaddress", "1.0.23") +#if matchcomp: +# print("matched comp") +#else: +# print("no comp match") +#if matchver: +# print("matched ver") +#else: +# print("no ver match") #comp_ver_url = create_cust_comp("MY COMPONENT z", "1", args.license_name) # #comp_url = "https://purl-validation.saas-staging.blackduck.com/api/components/886c04d4-28ce-4a27-be4c-f083e73a9f69" @@ -350,7 +499,7 @@ def add_to_sbom(proj_version_url, comp_ver_url): # "spdx_id": "SPDXRef-Pkg-react-bootstrap-2.1.2-30223", # "version": "2.1.2", # "origin": null -# TODO this try/except actually isn't right +# TODO this try/except isn't quite right try: outfile = open(args.out_file, 'w') except: logging.exception("Failed to open file for writing: " + args.out_file) @@ -368,6 +517,7 @@ def add_to_sbom(proj_version_url, comp_ver_url): assert len(projects) == 1, \ f"There should one project named {args.project_name}. Found {len(projects)}" project = projects[0] + # Fetch Version (can only have 1) params = { 'q': [f"versionName:{args.version_name}"] @@ -381,7 +531,6 @@ def add_to_sbom(proj_version_url, comp_ver_url): logging.debug(f"Found {project['name']}:{version['versionName']}") - # situations to consider + actions # 1) No purl available : check SBOM for comp+ver, then add cust comp + add to SBOM # 2) Have purl + found in KB @@ -417,7 +566,7 @@ def add_to_sbom(proj_version_url, comp_ver_url): # NOTE: BD can change the original component name # EX: "React" -> "React from Facebook" if package.external_references: - inkb, matchname = find_comp_in_kb(matchname, package.external_references[0].locator) + inkb, matchname, matchver = find_comp_in_kb(matchname, matchver, package.external_references[0].locator) if inkb: kb_matches += 1 else: nopurl += 1 @@ -426,9 +575,9 @@ def add_to_sbom(proj_version_url, comp_ver_url): print(" " + package.spdx_id) print(" " + package.version) - if find_comp_in_bom(bd, matchname, matchver, version): + if find_comp_in_bom(matchname, matchver, version): bom_matches += 1 - #print(" Found comp match in BOM: " + matchname + matchver) + print(" Found comp match in BOM: " + matchname + matchver) else: nomatch += 1 comp_data = { @@ -439,6 +588,11 @@ def add_to_sbom(proj_version_url, comp_ver_url): } comps_out.append(comp_data) + # TODO what about: KB exists but not in BOM?? + # find_cust_comp is not generic enough for that situation + #if inkb: + # TODO handle add KB match to BOM here, short-circuit steps below + # Check if custom component already exists comp_url, comp_ver_url = find_cust_comp(package.name, package.version) @@ -446,20 +600,19 @@ def add_to_sbom(proj_version_url, comp_ver_url): # Custom component did not exist, so create it cust_comp_count += 1 comp_ver_url = create_cust_comp(package.name, package.version, - args.license_name, approval) + args.license_name) elif comp_url and not comp_ver_url: # Custom component existed, but not the version we care about cust_ver_count += 1 print(f"Adding version {package.version} to custom component {package.name}") comp_ver_url = create_cust_comp_ver(comp_url, package.version, args.license_name) - # DEBUG - quit() else: print("Custom component already exists, not in SBOM") - # is this possible? i don't think so + # is this possible? assert(comp_ver_url), f"No comp_ver URL found for {package.name} {package.version}" - print(f"Adding component to SBOM: {package.name} {package.version}") + + print(f"Adding component to SBOM: {package.name} aka {matchname} {package.version}") add_to_sbom(proj_version_url, comp_ver_url) # Save unmatched components From 0492d95cc13748fbc753479761c7358c7c6d814a Mon Sep 17 00:00:00 2001 From: Makoto Koishi Date: Fri, 25 Aug 2023 15:21:11 +0900 Subject: [PATCH 019/146] New report example for component BOM and file BOM information is added. --- examples/client/consolidated_file_report.py | 820 ++++++++++++++++++++ 1 file changed, 820 insertions(+) create mode 100755 examples/client/consolidated_file_report.py diff --git a/examples/client/consolidated_file_report.py b/examples/client/consolidated_file_report.py new file mode 100755 index 00000000..1a7c36b6 --- /dev/null +++ b/examples/client/consolidated_file_report.py @@ -0,0 +1,820 @@ +''' +Created on August 23, 2023 + +@author: mkoishi + +Generate reports which consolidates information on BOM components, versions with license information, BOM files with +license and copyright information from KB and license search (discoveries - file licenses or file copyrights), +and BlackDuck unmatched files. + +Copyright (C) 2023 Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +''' + +import argparse +import logging +import sys +import os +import re +import time +import subprocess +import json +import traceback +import copy +import ijson +from json2html import * +from blackduck import Client +from zipfile import ZipFile + +program_description = \ +'''This script collects BlackDuck reports for version details and discoveries, and it generates new reports. +The newly generated reports are a report of BOM component-versions with license information, a report of BOM files that consolidates licenses and copyright texts information from KB and discoveries (e.g., file licenses, file copyrights), and a report of BlackDuck unmatched files. +Optionally, Synopsys-Detect can be executed from the script, and if it is selected, it runs in synchronous mode. Please consider adjusting the "detect.timeout" detect parameter because completion of Synopsys-Detect is estimated to take longer. + +Config file: +API Token, hub URL and two more options need to be placed in the .restconfig.json file that must be placed in the same folder where this script resides. +For more information, please find the instructions in the linked contents at https://community.synopsys.com/s/article/How-to-use-the-hub-rest-api-python-for-Black-Duck + +Pre-requisites: +1) python (>=3.5) and pip are installed. +2) Install PyPI modules "blackduck", "ijson" and "json2html. + +Examples: +1) If Synopsys-Detect is wanted to execute prior to the HTML report generation that includes file copyright texts, then +python3 ./consolidated_file_report.py \ +-f html -cl 2 -rr 100 \ +-dp detect.blackduck.signature.scanner.snippet.matching=SNIPPET_MATCHING \ +detect.blackduck.signature.scanner.license.search=true \ +detect.blackduck.signature.scanner.copyright.search=true \ +detect.source.path= \ +detect.timeout=1800 \ +blackduck.trust.cert=true + +Note: blackduck.trust.cert=true is not recommended in the production environment. + +2) If execution of Synopsys-Detect is wanted to bypass and no copyright texts are needed in the file report, then +python3 ./consolidated_file_report.py \ +-f html -sd -rr 100 -dp detect.source.path= + +Note: Please provide -dp detect.source.path parameter. If otherwise, the script has no clue for the target source folder. +''' + +# Synopsys Detect parameters +DOWNLOAD_DETECT = ["curl", "-s", "-L", "https://detect.synopsys.com/detect8.sh"] +BLACKDUCK_URL = "--blackduck.url=" +BLACKDUCK_TOKEN = "--blackduck.api.token=" +BLACKDUCK_PROJECT = "--detect.project.name=" +BLACKDUCK_VERSION = "--detect.project.version.name=" +BLACKDUCK_WAIT = "--detect.wait.for.results=true" +ENV_DETECT_VERSION = "DETECT_LATEST_RELEASE_VERSION" +# BD report general +BLACKDUCK_REPORT_MEDIATYPE = "application/vnd.blackducksoftware.report-4+json" +blackduck_report_download_api = "/api/projects/{projectId}/versions/{projectVersionId}/reports/{reportId}/download" +blackduck_link_component_ui_api = "/api/projects/{projectId}/versions/{projectVersionId}/components" +# BD version details report +blackduck_create_version_report_api = "/api/versions/{projectVersionId}/reports" +blackduck_version_report_filename = "./blackduck_version_report_for_{projectVersionId}.zip" +# BD discoveries report +blackduck_create_discoveries_report_api = "/api/versions/{projectVersionId}/license-reports" +blackduck_discoveries_report_filename = "./blackduck_discoveries_report_for_{projectVersionId}.zip" +# Consolidated report +BLACKDUCK_VERSION_MEDIATYPE = "application/vnd.blackducksoftware.status-4+json" +BLACKDUCK_VERSION_API = "/api/current-version" +BLACKDUCK_SOURCE_PATH = "detect.source.path" +REPORT_DIR = "./blackduck_consolidated_file_report" +REPORT_HEADER = "/header_report" +REPORT_OS_FILE = "/os_file_report" +REPORT_COMPONENT_BOM = "/component_bom_report" +REPORT_FILE_BOM = "/file_bom_report" +REPORT_DISCOVERY = "/discovery" +# Retries to wait for BD report creation. RETRY_LIMIT can be overwritten by the script parameter. +RETRY_LIMIT = 30 +RETRY_TIMER = 30 +# Reports +report_content = { + 'title' : 'Black Duck Consolidated File Report', + 'configurationSettings':{ + 'detectParameters': [], + 'scanDateTime': '', + 'blackDuckVersion': '', + 'linkToBlackDuckProjectVersionInUI': '' + }, + 'fileInventory': { + 'linkToUnmatchedOsFileData': "", + 'linkToBomComponentEntries': "", + 'linkToBomFileEntries': "", + } +} +report_os_file = { + 'unmatchedOsFileEntries': { + 'description': 'This list includes folders and files which belong to the target source directory and are not matched by BlackDuck', + 'unmatched': [] + } +} +report_component_bom = { + 'bomComponentEntries': { + 'description': 'This list includes BOM component information which is extracted from BlackDuck project version report.', + 'bomComponents': [] + } +} +report_file_bom = { + 'bomFileEntries': { + 'description': 'This list includes BOM file information which is extracted from BlackDuck project version report and Discovery report.', + 'bomFiles': [], + 'unmatchedFileDiscoveries': [] + } +} + +def log_config(debug): + if debug: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.DEBUG) + else: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.INFO) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("blackduck").setLevel(logging.WARNING) + +def parse_parameter(): + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("project", + metavar="project", + type=str, + help="Provide the BlackDuck project name.") + parser.add_argument("version", + metavar="version", + type=str, + help="Provide the BlackDuck project version name.") + parser.add_argument("-f", + "--report_format", + metavar="", + type=str, + default="json", + help="Specify the report format. Currently either JSON or HTML is supported. Default is JSON.") + parser.add_argument("-cl", + "--copyright_level", + metavar="", + type=int, + default=0, + help="Specify the included copyright text level. Level 0 is no copyright texts included, 1 is only copyright texts from KB included, 2 is copyright texts from KB and discoveries included.") + parser.add_argument("-sd", + "--skip_detect", + action='store_true', + help="Set if execution of Synopsys-Detect is wanted to bypass.") + parser.add_argument("-dv", + "--detect_version", + metavar="", + type=str, + default="latest", + help="Specify the Synopsys Detect version to download and run. If not set, the latest version will run.") + parser.add_argument("-rr", + "--report_retries", + metavar="", + type=int, + default=RETRY_LIMIT, + help="Retries for receiving the generated BlackDuck report. Timeout timer is hard-coded 30 sec. Generating a copyright report tends to take longer.") + parser.add_argument("-t", + "--timeout", + metavar="", + type=int, + default=15, + help="Timeout for REST-API. Some API may take longer than the default 15 seconds") + parser.add_argument("-r", + "--retries", + metavar="", + type=int, + default=3, + help="Retries for REST-API. Some API may need more retries than the default 3 times") + parser.add_argument("-dp", + "--detect_parameters", + metavar="", + type=str, + nargs="*", + default="", + help="List Synopsys Detect parameters with whitespace separators and without '--'. Example: -dp detect.blackduck.signature.scanner.snippet.matching=SNIPPET_MATCHING detect.blackduck.signature.scanner.license.search=true ") + return parser.parse_args() + +def run_detect(project, version, bd_url, bd_token, detect_version, bd_params=None): + """ Download and run Synopsys Detect. """ + # TODO: Consider to change to async and pall completion of the scan because that maybe more robust against network errors. + detect_params = [ + BLACKDUCK_URL + bd_url, + BLACKDUCK_TOKEN + bd_token, + BLACKDUCK_PROJECT + project, + BLACKDUCK_VERSION + version, + BLACKDUCK_WAIT] + if bd_params is not None: + for param in bd_params: + detect_params.append(f"--{param}") + + # Download the designated detect version of synopsys-detect + if detect_version != "latest": + major_version = detect_version.split(".", 1)[0] + for i, element in enumerate(DOWNLOAD_DETECT): + if element == "https://detect.synopsys.com/detect8.sh": + DOWNLOAD_DETECT[i] = element.replace("detect8", f"detect{major_version}") + break + my_env = os.environ.copy() + my_env[ENV_DETECT_VERSION] = detect_version + with open("synopsys-detect.sh", "w") as detect: + logging.info(f"Synopsys Detect version {detect_version} is being downloaded now!") + subprocess.run(DOWNLOAD_DETECT, stdout=detect) + + detect_command = ["bash", f"{detect.name}"] + detect_params + logging.info(f"synopsys-detect synchronous scan is in execution now!") + if detect_version != "latest": + results = subprocess.run(detect_command, capture_output=True, text=True, env=my_env) + else: + results = subprocess.run(detect_command, capture_output=True, text=True) + + print(results.stdout) + print(results.stderr) + results.check_returncode() + return + +def get_bd_project_data(hub_client, project_name, version_name): + """ Get and return project ID, version ID and codelocations. """ + project_id = "" + for project in hub_client.get_resource("projects"): + if project['name'] == project_name: + project_id = (project['_meta']['href']).split("projects/", 1)[1] + break + if project_id == "": + sys.exit(f"No project for {project_name} was found!") + version_id = codelocations = "" + for version in hub_client.get_resource("versions", project): + if version['versionName'] == version_name: + version_id = (version['_meta']['href']).split("versions/", 1)[1] + for link in version['_meta']['links']: + if link['rel'] == "codelocations": + codelocations = link['href'] + break + break + if version_id == "": + sys.exit(f"No project version for {version_name} was found!") + if codelocations == "": + sys.exit(f"No codelocations for {project_name} {version_name} found ") + + return project_id, version_id, codelocations + +def report_create(hub_client, url, body): + """ + Request BlackDuck to create report. Requested report is included in the request payload. + """ + res = hub_client.session.post(url, headers={'Content-Type': BLACKDUCK_REPORT_MEDIATYPE}, json=body) + if res.status_code != 201: + sys.exit(f"BlackDuck report creation failed with status {res.status_code}!") + # return report_url + return res.headers['Location'] + +def report_download(hub_client, report_url, project_id, version_id, retry_count): + """ + Download the generated report after the report completion. We will retry until reaching the retry-limit. + """ + retries = retry_count + while retries: + res = hub_client.session.get(report_url, headers={'Accept': BLACKDUCK_REPORT_MEDIATYPE}) + if res.status_code == 200 and (json.loads(res.content))['status'] == "COMPLETED": + report_id = report_url.split("reports/", 1)[1] + download_url = (((blackduck_report_download_api.replace("{projectId}", project_id)) + .replace("{projectVersionId}", version_id)) + .replace("{reportId}", report_id)) + res = hub_client.session.get(download_url, + headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + if res.status_code != 200: + sys.exit(f"BlackDuck report download failed with status {res.status_code} for {download_url}!") + return res.content + elif res.status_code != 200: + sys.exit(f"BlackDuck report creation not completed successfully with status {res.status_code}") + else: + logging.info(f"Waiting for the report generation for {report_url} with the remaining retries {retries} times.") + retries -= 1 + time.sleep(RETRY_TIMER) + sys.exit(f"BlackDuck report for {report_url} was not generated after retries {RETRY_TIMER} sec * {retry_count} times!") + +def get_version_detail_report(hub_client, project_id, version_id, retries): + """ Create and get BOM component and BOM source file report in json. """ + create_version_url = blackduck_create_version_report_api.replace("{projectVersionId}", version_id) + body = { + 'reportFormat' : 'JSON', + 'locale' : 'en_US', + 'versionId' : f'{version_id}', + 'categories' : [ 'COMPONENTS', 'FILES' ] + } + report_url = report_create(hub_client, create_version_url, body) + # Zipped report content is received and write the content to a local zip file + content = report_download(hub_client, report_url, project_id, version_id, retries) + output_file = blackduck_version_report_filename.replace("{projectVersionId}", version_id) + with open(output_file, "wb") as f: + f.write(content) + return output_file + +def get_discovery_report(hub_client, project_id, version_id, retries, copyright): + """ Create and get discovery report for licenses and copyrights in json. """ + create_discoveries_url = blackduck_create_discoveries_report_api.replace("{projectVersionId}", version_id) + body = { + 'reportFormat' : 'JSON', + 'locale' : 'en_US', + 'versionId' : f'{version_id}', + 'categories' : [ 'FILE_LICENSE_DATA', 'DEEP_LICENSE_DATA', 'UNMATCHED_FILE_DISCOVERIES' ] + } + if copyright == 1: + body['categories'].append('COPYRIGHT_TEXT') + elif copyright ==2: + body['categories'].extend(['COPYRIGHT_TEXT', 'FILE_COPYRIGHT_TEXT']) + report_url = report_create(hub_client, create_discoveries_url, body) + content = report_download(hub_client, report_url, project_id, version_id, retries) + output_file = blackduck_discoveries_report_filename.replace("{projectVersionId}", version_id) + with open(output_file, "wb") as f: + f.write(content) + return output_file + +def get_folder_size(path="."): + """ + Calculate the size of the given folder by summing up belonging files. + Remarks: Calculated folder size maybe shows different figure from OS command line because + OS commands may take the file system's chunk into consideration. + """ + total_size = 0 + with os.scandir(path) as it: + for entry in it: + if entry.is_file(): + size = entry.stat().st_size + total_size = total_size + size + elif entry.is_dir(): + size = get_folder_size(entry.path) + total_size = total_size + size + return total_size + +def get_os_path_for_unmatched(parent_dir, matched_paths): + """ + Traverse within the provided directory and yield OS folder or file data if it is not a matched path by BlackDuck. + """ + os_path_data = { + 'path': "", + 'sizeInBytes': 0 + } + os_path_stats = { + 'matched_folders': 0, + 'matched_files': 0, + 'unmatched_folders': 0, + 'unmatched_files': 0, + 'total_folders': 0, + 'total_files': 0 + } + + log_onerror = lambda err: logging.warn(f"An error reported during folder and file traverse. {str(err)}") + for path, folder_names, file_names in os.walk(parent_dir, onerror=log_onerror): + os_path_stats['total_folders'] = os_path_stats['total_folders'] + len(folder_names) + os_path_stats['total_files'] = os_path_stats['total_files'] + len(file_names) + + for folder_name in folder_names: + file_data = copy.deepcopy(os_path_data) + try: + abs_path = os.path.join(path, folder_name) + # Removal of parent path and add a slash for comparison with BlackDuck reported path + rel_path = abs_path.replace(f"{parent_dir}/", "") + "/" + if rel_path in matched_paths: + os_path_stats['matched_folders'] = os_path_stats['matched_folders'] + 1 + continue + file_data['path'] = rel_path + file_data['sizeInBytes'] = get_folder_size(abs_path) + os_path_stats['unmatched_folders'] = os_path_stats['unmatched_folders'] + 1 + # Keep the following log if something is wrong with the size + # logging.debug(f"filepath {file_data['path']} with size {file_data['sizeInBytes']} bytes") + yield file_data + except Exception as err: + logging.warning(f"An exception raised during OS folder report. {str(err)}") + continue + + for file_name in file_names: + file_data = copy.deepcopy(os_path_data) + try: + abs_path = os.path.join(path, file_name) + # Removal of parent path for comparison with BlackDuck reported file path + rel_path = abs_path.replace(f"{parent_dir}/", "") + if rel_path in matched_paths: + os_path_stats['matched_files'] = os_path_stats['matched_files'] + 1 + continue + file_data['path'] = rel_path + file_data['sizeInBytes'] = os.path.getsize(abs_path) + os_path_stats['unmatched_files'] = os_path_stats['unmatched_files'] + 1 + #logging.debug(f"filepath {file_data['path']} with size {file_data['sizeInBytes']} bytes") + yield file_data + except Exception as err: + logging.warning(f"An exception raised during OS file report. {str(err)}") + continue + + logging.debug(f"Matched folders: {os_path_stats['matched_folders']}, Matched files: {os_path_stats['matched_files']}, " + f"Unmatched folders: {os_path_stats['unmatched_folders']}, Unmatched files: {os_path_stats['unmatched_files']}, " + f"Total folders under the parent path: {os_path_stats['total_folders']}, " + f"Total files under the parent path: {os_path_stats['total_files']}, " + f"Number of matched paths by BlackDuck: {len(matched_paths)}") + return + +def pull_component_bom(component_bom): + """ + Extract BOM component data from BD project version report. + Remarks: Component BOM data includes components for SNIPPET matching which can outnumber the number of the components + in BlackDuck project version BOM UI. + """ + component_info = { + 'componentName': '', + 'componentVersionNames': [], + 'matchTypes': [], + 'licenses': [] + } + license_info = { + 'licenseType': '', + 'name': '', + 'licenseFamily': '', + 'licenses': [], + 'licenseDisplay': '' + } + license_nested = { + 'name': '', + 'licenseFamily': '', + 'licenses': [], + 'licenseDisplay': '' + } + + component_bom_data = copy.deepcopy(component_info) + component_bom_data['componentName'] = component_bom['producerProject']['name'] \ + if 'producerProject' in component_bom.keys() and 'name' in component_bom['producerProject'].keys() else "" + if 'producerReleases' in component_bom.keys(): + for release in component_bom['producerReleases']: + component_bom_data['componentVersionNames'].append(release['version']) + if 'matchTypes' in component_bom.keys(): + for match in component_bom['matchTypes']: + component_bom_data['matchTypes'].append(match) + if 'licenses' in component_bom.keys(): + for license in component_bom['licenses']: + license_data = copy.deepcopy(license_info) + license_data['licenseType'] = license['licenseType'] if 'licenseType' in license.keys() else "" + license_data['name'] = license['name'] if 'name' in license.keys() else "" + license_data['licenseFamily'] = license['codeSharing'] if 'codeSharing' in license.keys() else "" + if 'licenses' in license.keys(): + for in_license in license['licenses']: + nested_license_data = copy.deepcopy(license_nested) + nested_license_data['name'] = in_license['name'] if 'name' in in_license.keys() else "" + nested_license_data['licenseFamily'] = in_license['codeSharing'] if 'codeSharing' in in_license.keys() else "" + nested_license_data['licenseDisplay'] = in_license['licenseDisplay'] if 'licenseDisplay' in in_license.keys() else "" + nested_license_data['licenses'] = in_license['licenses'] if 'licenses' in in_license.keys() else [] + license_data['licenses'].append(nested_license_data) + license_data['licenseDisplay']= license['licenseDisplay'] if 'licenseDisplay' in license.keys() else "" + component_bom_data['licenses'].append(license_data) + return component_bom_data + +def pull_file_bom(file_bom): + """ + Extract BOM file data from BD project version report. + Remarks: If reported path is ended with '/', then that is folder path. If not, that is file path. + """ + + file_info = { + 'path' : '', + 'archiveContext' : '', + 'projectName' : '', + 'projectVersion' : '', + 'channelReleaseExternalNamespace' : '', + 'channelReleaseExternalId' : '', + 'matchType' : '', + 'snippetReviewStatus' : '', + 'licenses' : [], + 'copyrights' : { + 'copyrightTexts' : [], + 'fileCopyrightTexts' : [] + } + } + + file_bom_data = copy.deepcopy(file_info) + file_bom_data['path'] = file_bom['path'] if 'path' in file_bom.keys() else "" + file_bom_data['archiveContext'] = file_bom['archiveContext'] if 'archiveContext' in file_bom.keys() else "" + file_bom_data['projectName'] = file_bom['projectName'] if 'projectName' in file_bom.keys() else "" + file_bom_data['projectVersion'] = file_bom['version'] if 'version' in file_bom.keys() else "" + file_bom_data['matchType'] = file_bom['matchType'] if 'matchType' in file_bom.keys() else "" + if 'matchType' in file_bom.keys() and file_bom['matchType'] == "SNIPPET": + file_bom_data['snippetReviewStatus'] = file_bom['snippetReviewStatus'] + file_bom_data['channelReleaseExternalNamespace'] = file_bom['channelReleaseExternalNamespace'] \ + if 'channelReleaseExternalNamespace' in file_bom.keys() else "" + file_bom_data['channelReleaseExternalId'] = file_bom['channelReleaseExternalId'] \ + if 'channelReleaseExternalId' in file_bom.keys() else "" + return file_bom_data + +def pull_discovery_licenses(comp_license): + """ + Extract component licenses from discoveries. + """ + comp_license_info = { + 'projectName' : '', + 'versionName' : '', + 'licenses' : [] + } + license_info = { + 'name' : '', + 'sources' : [] + } + comp_license_data = copy.deepcopy(comp_license_info) + comp_license_data['projectName'] = comp_license['component']['projectName'] \ + if 'component' in comp_license.keys() and 'projectName' in comp_license['component'].keys() else "" + comp_license_data['versionName'] = comp_license['component']['versionName'] \ + if 'component' in comp_license.keys() and 'versionName' in comp_license['component'].keys() else "" + if 'licenses' in comp_license.keys(): + for license in comp_license['licenses']: + license_data = copy.deepcopy(license_info) + license_data['name'] = license['name'] if 'name' in license.keys() else "" + if 'sources' in license.keys(): + for source in license['sources']: + license_data['sources'].append(source) + comp_license_data['licenses'].append(license_data) + return comp_license_data + +def pull_discovery_copyrights(comp_copyright): + """ + Extract component copyrights from discoveries. + """ + comp_copyright_info = { + 'originFullName' : '', + 'copyrights' : { + 'copyrightTexts' : [], + 'fileCopyrightTexts' : [] + } + } + comp_copyright_data = copy.deepcopy(comp_copyright_info) + comp_copyright_data['originFullName'] = comp_copyright['originFullName'] if 'originFullName' in comp_copyright.keys() else "" + if 'copyrightTexts' in comp_copyright.keys(): + for copyright in comp_copyright['copyrightTexts']: + comp_copyright_data['copyrights']['copyrightTexts'].append(copyright) + if 'fileCopyrightTexts' in comp_copyright.keys(): + for file_copyright in comp_copyright['fileCopyrightTexts']: + comp_copyright_data['copyrights']['fileCopyrightTexts'].append(file_copyright) + return comp_copyright_data + +def pull_discovery_unmatched(comp_unmatched): + """ + Extract unmatched files from discoveries. + """ + comp_unmatched_info = { + 'fileNames' : [], + 'resourceName' : '', + 'matchType' : '', + 'matchTypeLabel' : '' + } + file_name_info = { + 'path' : '', + 'archiveContext' : '', + 'compositePathContext' : '', + 'fileName' : '' + } + + comp_unmatched_data = copy.deepcopy(comp_unmatched_info) + if 'fileNames' in comp_unmatched.keys(): + for file_name in comp_unmatched['fileNames']: + file_name_data = copy.deepcopy(file_name_info) + file_name_data['path'] = file_name['path'] if 'path' in file_name.keys() else "" + file_name_data['archiveContext'] = file_name['archiveContext'] \ + if 'archiveContext' in file_name.keys() else "" + file_name_data['compositePathContext'] = file_name['compositePathContext'] \ + if 'compositePathContext' in file_name.keys() else "" + file_name_data['fileName'] = file_name['fileName'] if 'fileName' in file_name.keys() else "" + comp_unmatched_data['fileNames'].append(file_name_data) + comp_unmatched_data['resourceName'] = comp_unmatched['resourceName'] if 'resourceName' in comp_unmatched.keys() else "" + comp_unmatched_data['matchType'] = comp_unmatched['matchType'] if 'matchType' in comp_unmatched.keys() else "" + comp_unmatched_data['matchTypeLabel'] = comp_unmatched['matchTypeLabel'] if 'matchTypeLabel' in comp_unmatched.keys() else "" + return comp_unmatched_data + +def get_scanned_time(hub_client, codelocations): + """ Retrieve the scanned time from the codelocations data. """ + res = hub_client.session.get(codelocations) + if res.status_code == 200 and res.content: + return json.loads(res.content)['items'][0]['updatedAt'] + # Let's use the 1st element of the list because multiple scans should have taken place at the same time + else: + sys.exit(f"Get codelocations failed for codelocations {codelocations} with status {res.status_code}") + +def get_blackduck_version(hub_client): + url = hub_client.base_url + BLACKDUCK_VERSION_API + res = hub_client.session.get(url) + if res.status_code == 200 and res.content: + return json.loads(res.content)['version'] + else: + sys.exit(f"Get BlackDuck version failed with status {res.status_code}") + +def generate_file_report(hub_client, project_id, version_id, codelocations, copyright_level, format, retries, detect_params=None): + """ + Create a consolidated file report from BlackDuck project version report and notice report. + Remarks: + """ + if not os.path.exists(REPORT_DIR): + os.makedirs(REPORT_DIR) + + # Report headers + report_content['configurationSettings']['scanDateTime'] = get_scanned_time(hub_client, codelocations) + report_content['configurationSettings']['blackDuckVersion'] = get_blackduck_version(hub_client) + report_content['configurationSettings']['detectParameters'] = detect_params + blackduck_link_component_ui_api.replace("{projectId}", project_id).replace("{projectVersionId}", version_id) + report_content['configurationSettings']['linkToBlackDuckProjectVersionInUI'] = \ + hub_client.base_url + blackduck_link_component_ui_api.replace("{projectId}", project_id).replace("{projectVersionId}", version_id) + + # Report body - Component BOM, file BOM with Discoveries data + version_report_zip = get_version_detail_report(hub_client, project_id, version_id, retries) + with ZipFile(f"./{version_report_zip}", "r") as vzf: + vzf.extractall() + for i, unzipped_version in enumerate(vzf.namelist()): + if re.search(r"\bversion.+json\b", unzipped_version) is not None: + break + if i + 1 >= len(vzf.namelist()): + sys.exit(f"Version detail file not found in the downloaded report: {version_report_zip}!") + + # Report body - Component BOM report + # Iterated json handling to reduce memory consumption. Remarks: Divided to two file-open sessions for components and files + # with respective file handles, because if otherwise ijson crashes! + with open(f"./{unzipped_version}", "r") as uvf: + for i, comp_bom in enumerate(ijson.items(uvf, 'aggregateBomViewEntries.item')): + comp_data = pull_component_bom(comp_bom) + report_component_bom['bomComponentEntries']['bomComponents'].append(comp_data) + logging.info(f"Number of the reported components {i+1}") + with open(REPORT_DIR + REPORT_COMPONENT_BOM + f".{format}", "w") as cmf: + if format == "json": + cmf.write(json.dumps(report_component_bom)) + else: + cmf.write(json2html.convert(json = json.dumps(report_component_bom))) + report_content['fileInventory']['linkToBomComponentEntries'] = \ + "file://" + os.path.abspath(REPORT_DIR + REPORT_COMPONENT_BOM + f".{format}") + + # Discovery data - licenses, copyrights and unmatched files - is fetched and integrated with file BOM report. + discovery_data = {'discoveries': {'licenses': [], 'copyrights': []}} + discovery_report_zip = get_discovery_report(hub_client, project_id, version_id, retries, copyright_level) + with ZipFile(f"./{discovery_report_zip}", "r") as zlf: + zlf.extractall() + for i, unzipped_discovery in enumerate(zlf.namelist()): + if re.search(r"\bversion-license.+json\b", unzipped_discovery) is not None: + break + if i + 1 >= len(zlf.namelist()): + sys.exit(f"License file not found in downloaded report: {discovery_report_zip}!") + # Do not reuse TextIoWrapper for ijson to handle multiple items. If do, ijson crashes. + with open(f"./{unzipped_discovery}", "r") as ulf: + for i, comp_license in enumerate(ijson.items(ulf, 'componentLicenses.item')): + comp_license_data = pull_discovery_licenses(comp_license) + discovery_data['discoveries']['licenses'].append(comp_license_data) + logging.info(f"Number of the reported discovery licenses {i+1}") + if copyright_level != 0: + with open(f"./{unzipped_discovery}", "r") as ucf: + for i, comp_copyright in enumerate(ijson.items(ucf, 'componentCopyrightTexts.item')): + comp_copyright_data = pull_discovery_copyrights(comp_copyright) + discovery_data['discoveries']['copyrights'].append(comp_copyright_data) + logging.info(f"Number of the reported discovery copyright texts {i+1}") + with open(f"./{unzipped_discovery}", "r") as uuf: + for i, comp_unmatched in enumerate(ijson.items(uuf, 'unmatchedFileData.item')): + comp_unmatched_data = pull_discovery_unmatched(comp_unmatched) + report_file_bom['bomFileEntries']['unmatchedFileDiscoveries'].append(comp_unmatched_data) + logging.info(f"Number of the reported discovery unmatched files texts {i+1}") + + # Report body - Generate file BOM report. Discovery data is integrated. + with open(f"./{unzipped_version}", "r") as uvf: + for i, file_bom in enumerate(ijson.items(uvf, 'detailedFileBomViewEntries.item')): + file_data = pull_file_bom(file_bom) + disc_licenses = list(filter(lambda license_x: + license_x['projectName'] == file_data['projectName'] and + license_x['versionName'] == file_data['projectVersion'], + discovery_data['discoveries']['licenses'])) + if len(disc_licenses) != 0: + disc_license = disc_licenses[0] + for license in disc_license['licenses']: + file_data['licenses'].append(license) + else: + logging.debug(f"No discovery license found for file with this component: {file_data['projectName']} \ + and {file_data['projectVersion']}") + if len(disc_licenses) > 1: + logging.warning(f"More than one discovery license: {disc_licenses} found for \ + {file_data['projectName']}" and file_data['projectVersion']) + + if copyright_level != 0: + origin_name = file_data['channelReleaseExternalNamespace'] + ":" + file_data['channelReleaseExternalId'] + disc_copyrights = list(filter(lambda copyright_x: + copyright_x['originFullName'] == origin_name, + discovery_data['discoveries']['copyrights'])) + if len(disc_copyrights) != 0: + disc_copyright = disc_copyrights[0] + for copyright in disc_copyright['copyrights']['copyrightTexts']: + file_data['copyrights']['copyrightTexts'].append(copyright) + for file_copyright in disc_copyright['copyrights']['fileCopyrightTexts']: + file_data['copyrights']['fileCopyrightTexts'].append(file_copyright) + else: + logging.debug(f"No discovery copyright found for file with this origin: {origin_name}") + if len(disc_copyrights) > 1: + logging.warning(f"More than one discovery copyright: {disc_copyrights} found \ + for the same origin {origin_name}") + + report_file_bom['bomFileEntries']['bomFiles'].append(file_data) + logging.info(f"Number of the reported files {i+1}") + + with open(REPORT_DIR + REPORT_FILE_BOM + f".{format}", "w") as flf: + if format == "json": + flf.write(json.dumps(report_file_bom)) + else: + flf.write(json2html.convert(json = json.dumps(report_file_bom))) + report_content['fileInventory']['linkToBomFileEntries'] = \ + "file://" + os.path.abspath(REPORT_DIR + REPORT_FILE_BOM + f".{format}") + + # Report body - Paths and sizes for folders and files which are not matched by BlackDuck + parent_path = "." + for param in detect_params: + if re.search(rf"{BLACKDUCK_SOURCE_PATH}", param) is not None: + parent_path = param.split("=", 1)[1] + break + logging.info("OS file information for the target source is being traversed and reported.") + matched_paths = [] + for file_bom in report_file_bom['bomFileEntries']['bomFiles']: + # BlackDuck can match internal folders or files within archive files, e.g., .jar or .zip or .tar.gz. + # OS unmatched file function does not search inside of the archived contents as it sounds too much. + if (file_bom['archiveContext']) != "": + continue + else: + matched_paths.append(file_bom['path']) + # Let's remove duplicated path entries. BD may report multiple BOM file entries for the same path. + unique_paths = list(set(matched_paths)) + if len(unique_paths) != len(matched_paths): + logging.warning("BlackDuck component BOM contains duplicated file paths!") + for os_path_data in get_os_path_for_unmatched(parent_path, unique_paths): + report_os_file['unmatchedOsFileEntries']['unmatched'].append(os_path_data) + with open(REPORT_DIR + REPORT_OS_FILE + f".{format}", "w") as osf: + if format == "json": + osf.write(json.dumps(report_os_file)) + else: + osf.write(json2html.convert(json = json.dumps(report_os_file))) + report_content['fileInventory']['linkToUnmatchedOsFileData'] = \ + "file://" + os.path.abspath(REPORT_DIR + REPORT_OS_FILE + f".{format}") + + with open(REPORT_DIR + REPORT_HEADER + f".{format}", "w") as rf: + if format == "json": + rf.write(json.dumps(report_content)) + else: + rf.write(json2html.convert(json = json.dumps(report_content))) + +def main(): + args = parse_parameter() + + try: + if args.copyright_level >= 3: + sys.exit("please provide the copyright level which is either 0 or 1 or 2!") + if (args.report_format).lower() != "json" and (args.report_format).lower() != "html": + sys.exit("Please set either 'json' or 'html' to the report format") + + with open(".restconfig.json", "r") as f: + config = json.load(f) + # Remove last slash if there is, otherwise REST API may fail. + if re.search(r".+/$", config['baseurl']): + bd_url = config['baseurl'][:-1] + else: + bd_url = config['baseurl'] + bd_token = config['api_token'] + bd_insecure = config['insecure'] + if config['debug']: + debug = 1 + + log_config(debug) + + if not args.skip_detect: + run_detect(args.project, args.version, bd_url, bd_token, args.detect_version, args.detect_parameters) + + hub_client = Client(token=bd_token, + base_url=bd_url, + verify=bd_insecure, + timeout=args.timeout, + retries=args.retries) + + project_id, version_id, codelocations = get_bd_project_data(hub_client, args.project, args.version) + + generate_file_report(hub_client, + project_id, + version_id, + codelocations, + args.copyright_level, + args.report_format.lower(), + args.report_retries, + args.detect_parameters + ) + + except (Exception, BaseException) as err: + logging.error(f"Exception by {str(err)}. See the stack trace") + traceback.print_exc() + +if __name__ == '__main__': + sys.exit(main()) From a84ed4eb45aa33ded89c71692d9ff07bfa3a8936 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Fri, 25 Aug 2023 16:16:19 -0400 Subject: [PATCH 020/146] Debugging/cleanup/refactor. Add functionality to handle a situation where KB match was successful, but component was not located in BOM --- examples/client/parse_spdx.py | 191 ++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 92 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 06655a21..9efc2814 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -39,7 +39,7 @@ Requirements - python3 version 3.8 or newer recommended -- the following packages are used by the script and should be installed +- The following packages are used by the script and should be installed prior to use: argparse blackduck @@ -60,9 +60,14 @@ pip3 install argparse blackduck sys logging time json spdx_tools -usage: parse_spdx.py [-h] --base-url BASE_URL --token-file TOKEN_FILE --spdx-file SPDX_FILE --out-file OUT_FILE --project PROJECT_NAME --version VERSION_NAME [--no-verify] +usage: parse_spdx.py [-h] --base-url BASE_URL --token-file TOKEN_FILE + --spdx-file SPDX_FILE --out-file OUT_FILE --project + PROJECT_NAME --version VERSION_NAME + [--license LICENSE_NAME] [--no-verify] + [--no-spdx-validate] -Parse SPDX file and verify if component names are in current SBOM for given project-version +Parse SPDX file and verify if component names are in current SBOM for given +project-version optional arguments: -h, --help show this help message and exit @@ -76,7 +81,11 @@ Project that contains the BOM components --version VERSION_NAME Version that contains the BOM components + --license LICENSE_NAME + License name to use for custom components (default: + NOASSERTION) --no-verify Disable TLS certificate verification + --no-spdx-validate Disable SPDX validation ''' @@ -145,6 +154,10 @@ def poll_for_upload(sbom_name): sleep_time = 10 matched_scan = False + # TODO also check for api/projects//versions//codelocations + # -- status - operationNameCode = ServerScanning, operationName=Scanning, status + # -- should be COMPLETED, not IN_PROGRESS + # -- operatinName: Scanning # Search for the latest scan matching our SBOM # This might be a risk for a race condition params = { @@ -212,10 +225,10 @@ def upload_sbom_file(filename, project, version): files = {"file": (filename, open(filename,"rb"), mime_type)} fields = {"projectName": project, "versionName": version} response = bd.session.post("/api/scan/data", files = files, data=fields) - logging.info(response) + logging.debug(response) if response.status_code == 409: - logging.info(f"File {filename} is already mapped to a different project version") + logging.error(f"File {filename} is already mapped to a different project version") if response.status_code != 201: logging.error(f"Failed to upload SPDX file:") @@ -227,41 +240,23 @@ def upload_sbom_file(filename, project, version): # Lookup the given pURL in the BD KB. -# If successfully matched, update the associated package name and version with the data from the KB. -# This will improve the accuracy of later lookups. We are replacing the SPDX input data with the -# data stored in the KB. # # Inputs: -# matchname - Name of package from the SPDX input file -# matchver - Version of package -# extref - pURL to look up +# extref - pURL to look up # # Returns: -# purlmatch - boolean (True if successful KB lookup) -# matchname - Original parameter OR updated to reflect KB lookup name -# matchver - Original parameter OR updated to reflect KB lookup version -def find_comp_in_kb(matchname, matchver, extref): - purlmatch = False +# If match: API matching data (the "result" object) +# No match: None +def find_comp_in_kb(extref): params = { 'packageUrl': extref } for result in bd.get_items("/api/search/purl-components", params=params): - # This query should result in exactly 1 match - purlmatch = True - # Override the spdx name and use the known KB name - if matchname != result['componentName']: - print(f"Renaming {matchname} -> {result['componentName']}") - matchname = result['componentName'] - # Override the spdx version and use the string from KB - # for example, v2.8.5 -> 2.8.5 - if matchver != result['versionName']: - print(f"Renaming {matchver} -> {result['versionName']}") - matchver = result['versionName'] + # Should be exactly 1 match when successful + return(result) - return(purlmatch, matchname, matchver) - - # fall through -- lookup failed, so we keep the original name/ver - return(purlmatch, matchname, matchver) + # Fall through -- lookup failed + return(None) # Locate component name + version in BOM @@ -472,8 +467,13 @@ def add_to_sbom(proj_version_url, comp_ver_url): # This will exit if it fails poll_for_upload(document.creation_info.name) -# some little debug/test stubs +# some debug/test stubs # TODO: delete these +#ver="https://purl-validation.saas-staging.blackduck.com/api/projects/c2b4463f-7996-4c45-8443-b69b4f82ef1d/versions/67e4f6f5-2f42-42c4-9b69-e39bad55f907" +#comp = "https://purl-validation.saas-staging.blackduck.com/api/components/fc0a76fe-70a4-4afa-9a94-c3c22d63454f/versions/fabaabb9-3b9a-4b5f-850a-39fe84c4cfc4" +#add_to_sbom(ver, comp) +#quit() + #matchcomp, matchver = find_cust_comp("ipaddress", "1.0.23") #if matchcomp: # print("matched comp") @@ -493,12 +493,8 @@ def add_to_sbom(proj_version_url, comp_ver_url): #add_to_sbom(pv, cv) #quit() -# Open unmatched component file -# Will save name, spdxid, version, and origin/purl for later in json format: -# "name": "react-bootstrap", -# "spdx_id": "SPDXRef-Pkg-react-bootstrap-2.1.2-30223", -# "version": "2.1.2", -# "origin": null +# Open unmatched component file to save name, spdxid, version, and +# origin/purl for later in json format # TODO this try/except isn't quite right try: outfile = open(args.out_file, 'w') except: @@ -531,16 +527,6 @@ def add_to_sbom(proj_version_url, comp_ver_url): logging.debug(f"Found {project['name']}:{version['versionName']}") -# situations to consider + actions -# 1) No purl available : check SBOM for comp+ver, then add cust comp + add to SBOM -# 2) Have purl + found in KB -# - In SBOM? -> done -# - Else -> add known KB comp to SBOM -# *** this shouldn't happen in theory -# 3) Have purl + not in KB (main case we are concerned with) -# - In SBOM? (maybe already added or whatever?) -> done -# - Else -> add cust comp + add to SBOM (same as 1) - # Stats to track bom_matches = 0 kb_matches = 0 @@ -560,60 +546,81 @@ def add_to_sbom(proj_version_url, comp_ver_url): purlmatch = False matchname = package.name matchver = package.version + print(f"Processing SPDX package: {matchname} {matchver}....") # Tracking unique package name + version from spdx file packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 - # NOTE: BD can change the original component name - # EX: "React" -> "React from Facebook" if package.external_references: - inkb, matchname, matchver = find_comp_in_kb(matchname, matchver, package.external_references[0].locator) - if inkb: kb_matches += 1 + # TODO need to handle the possiblity of: + # A) multiple extrefs + # B) an extref that is not a purl + # -- referenceType should be "purl" - ignore others? + kb_match = find_comp_in_kb(package.external_references[0].locator) + if (kb_match): + # Update package name and version to reflect the KB name/ver + print(f" KB match for {package.name} {package.version}") + kb_matches+=1 + matchname = kb_match['componentName'] + matchver = kb_match['versionName'] + else: + print(f" No KB match for {package.name} {package.version}") else: nopurl += 1 - print("No pURL found for component: ") - print(" " + package.name) - print(" " + package.spdx_id) - print(" " + package.version) + kb_match = None + print(f"No pURL provided for {package.name} {package.version}") if find_comp_in_bom(matchname, matchver, version): bom_matches += 1 - print(" Found comp match in BOM: " + matchname + matchver) + print(f" Found component in BOM: {matchname} {matchver}") + # It's in the BOM so we are happy + # Everything else below is related to adding to the BOM + continue + + # If we've gotten this far, the package is not in the BOM. + # Now we need to figure out: + # - Is it already in the KB and we need to add it? (should be rare) + # - Do we need to add a custom component? + # - Do we need to add a version to an existing custom component? + nomatch += 1 + print(f" Not present in BOM: {matchname} {matchver}") + comp_data = { + "name": package.name, + "spdx_id": package.spdx_id, + "version": package.version, + "origin": extref + } + comps_out.append(comp_data) + + # KB match was successful, but it wasn't in the BOM for some reason + if kb_match: + print(f" WARNING: {matchname} {matchver} in KB but not in SBOM") + add_to_sbom(proj_version_url, kb_match['version']) + # temp debug to find this case + quit() + # short-circuit the rest + continue + + # Check if custom component already exists + comp_url, comp_ver_url = find_cust_comp(package.name, package.version) + + if not comp_url: + # Custom component did not exist, so create it + cust_comp_count += 1 + comp_ver_url = create_cust_comp(package.name, package.version, + args.license_name) + elif comp_url and not comp_ver_url: + # Custom component existed, but not the version we care about + cust_ver_count += 1 + print(f" Adding version {package.version} to custom component {package.name}") + comp_ver_url = create_cust_comp_ver(comp_url, package.version, args.license_name) else: - nomatch += 1 - comp_data = { - "name": package.name, - "spdx_id": package.spdx_id, - "version": package.version, - "origin": extref - } - comps_out.append(comp_data) - - # TODO what about: KB exists but not in BOM?? - # find_cust_comp is not generic enough for that situation - #if inkb: - # TODO handle add KB match to BOM here, short-circuit steps below + print(" Custom component already exists, not in SBOM") - # Check if custom component already exists - comp_url, comp_ver_url = find_cust_comp(package.name, package.version) - - if not comp_url: - # Custom component did not exist, so create it - cust_comp_count += 1 - comp_ver_url = create_cust_comp(package.name, package.version, - args.license_name) - elif comp_url and not comp_ver_url: - # Custom component existed, but not the version we care about - cust_ver_count += 1 - print(f"Adding version {package.version} to custom component {package.name}") - comp_ver_url = create_cust_comp_ver(comp_url, package.version, args.license_name) - else: - print("Custom component already exists, not in SBOM") + # Shouldn't be possible + assert(comp_ver_url), f"No component URL found for {package.name} {package.version}" - # is this possible? - assert(comp_ver_url), f"No comp_ver URL found for {package.name} {package.version}" - - print(f"Adding component to SBOM: {package.name} aka {matchname} {package.version}") - add_to_sbom(proj_version_url, comp_ver_url) + print(f" Adding component to SBOM: {package.name} aka {matchname} {package.version}") + add_to_sbom(proj_version_url, comp_ver_url) # Save unmatched components json.dump(comps_out, outfile) @@ -622,9 +629,9 @@ def add_to_sbom(proj_version_url, comp_ver_url): print("\nStats: ") print("------") print(f" SPDX packages processed: {package_count}") -print(f" Non matches: {nomatch}") -print(f" KB matches: {kb_matches}") +print(f" Packages missing from BOM: {nomatch}") print(f" BOM matches: {bom_matches}") +print(f" KB matches: {kb_matches}") print(f" Packages missing purl: {nopurl}") print(f" Custom components created: {cust_comp_count}") print(f" Custom component versions created: {cust_ver_count}") From 678fb1e3490c9bfb6ab3f702d35683131414edd1 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Fri, 25 Aug 2023 17:26:12 -0400 Subject: [PATCH 021/146] note a situation that needs to get fixed in scan matching. added code to handle multiple ext refs more cleanly / non purl extrefs --- examples/client/parse_spdx.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 9efc2814..96a32387 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -160,6 +160,9 @@ def poll_for_upload(sbom_name): # -- operatinName: Scanning # Search for the latest scan matching our SBOM # This might be a risk for a race condition + # TODO Annoyingly, the sbom_name is not necessarily precisely our document + # name! Found a case where BD swaps a space for a "-" in the + # document name. params = { 'q': [f"name:{sbom_name}"], 'sort': ["updatedAt: ASC"] @@ -551,11 +554,20 @@ def add_to_sbom(proj_version_url, comp_ver_url): packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 if package.external_references: - # TODO need to handle the possiblity of: - # A) multiple extrefs - # B) an extref that is not a purl - # -- referenceType should be "purl" - ignore others? - kb_match = find_comp_in_kb(package.external_references[0].locator) + foundpurl = False + kb_match = None + for ref in package.external_references: + # There can be multiple extrefs - try to locate a purl + if (ref.reference_type == "purl"): + # TODO are we guaranteed only 1 purl? + # what would it mean to have >1? + foundpurl = True + kb_match = find_comp_in_kb(ref.locator) + extref = ref.locator + break + if not foundpurl: + nopurl += 1 + print(f" No pURL provided for {package.name} {package.version}") if (kb_match): # Update package name and version to reflect the KB name/ver print(f" KB match for {package.name} {package.version}") From 28f3599ce493c76fcdcd265c08b4a756e328c67f Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Fri, 25 Aug 2023 17:52:46 -0400 Subject: [PATCH 022/146] minor formatting/whitespace fixup --- examples/client/parse_spdx.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 96a32387..3c7408a5 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -211,7 +211,7 @@ def poll_for_upload(sbom_name): logging.error(f"No scan found for SBOM: {sbom_name}") else: logging.error(f"Unable to verify successful scan of SBOM: {sbom_name}") - + # If we got this far, it's a fatal error. sys.exit(1) # TODO do we care about project_groups? @@ -241,7 +241,6 @@ def upload_sbom_file(filename, project, version): logging.error(f"Status code {response.status_code}") sys.exit(1) - # Lookup the given pURL in the BD KB. # # Inputs: @@ -261,7 +260,6 @@ def find_comp_in_kb(extref): # Fall through -- lookup failed return(None) - # Locate component name + version in BOM # Inputs: # compname - Component name to locate @@ -295,7 +293,6 @@ def find_comp_in_bom(compname, compver, projver): return False return False - # Verifies if a custom component and version already exist in the system # # Inputs: @@ -386,7 +383,6 @@ def create_cust_comp(name, version, license): for version in bd.get_items(response.links['versions']['url']): return(version['_meta']['href']) - # Create a version for a custom component that already exists # # Inputs: @@ -410,7 +406,6 @@ def create_cust_comp_ver(comp_url, version, license): logging.error(f"Version {version} already exists for component") sys.exit(1) - # necessary? if response.status_code != 201: logging.error(f"Failed to add Version {version} to component") sys.exit(1) From bf8d5fd2e8361df44acc6645f18a89139f9ae71a Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Mon, 28 Aug 2023 14:54:32 -0400 Subject: [PATCH 023/146] Force custom components and comparisons to use all lowercase. Clean up some stray comments. Make a variable initialization cleaner. Improve some error messages --- examples/client/parse_spdx.py | 62 +++++++++++------------------------ 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 3c7408a5..9a6fb963 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -127,7 +127,6 @@ def spdx_validate(document): # TODO is there a way to distinguish between something fatal and something # BD can deal with? - # TODO - this can take forever, so add an optional --skip-validation flag for validation_message in validation_messages: # Just printing these messages intead of exiting. Later when we try to import # the file to BD, let's plan to exit if it fails. Seeing lots of errors in the @@ -162,7 +161,7 @@ def poll_for_upload(sbom_name): # This might be a risk for a race condition # TODO Annoyingly, the sbom_name is not necessarily precisely our document # name! Found a case where BD swaps a space for a "-" in the - # document name. + # document name. Need to be more general in the match. params = { 'q': [f"name:{sbom_name}"], 'sort': ["updatedAt: ASC"] @@ -280,12 +279,12 @@ def find_comp_in_bom(compname, compver, projver): # Search BOM for specific component name comps = bd.get_resource('components', projver, params=params) for comp in comps: - if comp['componentName'] != compname: + if comp['componentName'].lower() != compname.lower(): # The BD API search is inexact. Force our match to be precise. continue # Check component name + version name try: - if comp['componentVersionName'] == compver: + if comp['componentVersionName'].lower() == compver.lower(): return True except: # Handle situation where it's missing the version name for some reason @@ -293,7 +292,7 @@ def find_comp_in_bom(compname, compver, projver): return False return False -# Verifies if a custom component and version already exist in the system +# Verifies if a custom component and version already exist in the system. # # Inputs: # compname - Component name to locate @@ -303,7 +302,7 @@ def find_comp_in_bom(compname, compver, projver): # VerMatch - Contains matched component verison url, None for no match def find_cust_comp(compname, compver): params = { - 'q': [f"name:{compname}"] + 'q': [f"name:{compname.lower()}"] } matched_comp = None @@ -311,7 +310,7 @@ def find_cust_comp(compname, compver): # Warning: Relies on internal header headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} for comp in bd.get_resource('components', params=params, headers=headers): - if compname == comp['name']: + if compname.lower() == comp['name'].lower(): # Force exact match matched_comp = comp['_meta']['href'] else: @@ -320,7 +319,7 @@ def find_cust_comp(compname, compver): # Check version for version in bd.get_resource('versions', comp): - if compver == version['versionName']: + if compver.lower() == version['versionName'].lower(): # Successfully matched both name and version matched_ver = version['_meta']['href'] return(matched_comp, matched_ver) @@ -348,7 +347,10 @@ def get_license_url(license_name): logging.error(f"Failed to find license {license_name}") sys.exit(1) -# Create a custom component +# Create a custom component. The Name and Version strings are converted to +# lowercase strings to ensure a reliable experience (avoiding dup names +# with varying CapItaliZation) +# # Inputs: # name - Name of component to add # version - Version of component to add @@ -358,7 +360,7 @@ def create_cust_comp(name, version, license): print(f"Adding custom component: {name} {version}") license_url = get_license_url(license) data = { - 'name': name, + 'name': name.lower(), 'version' : { 'versionName' : version, 'license' : { @@ -383,7 +385,8 @@ def create_cust_comp(name, version, license): for version in bd.get_items(response.links['versions']['url']): return(version['_meta']['href']) -# Create a version for a custom component that already exists +# Create a version for a custom component that already exists. +# Force the version string to be lowercase. # # Inputs: # comp_url - API URL of the component to update @@ -394,7 +397,7 @@ def create_cust_comp(name, version, license): def create_cust_comp_ver(comp_url, version, license): license_url = get_license_url(license) data = { - 'versionName' : version, + 'versionName' : version.lower(), 'license' : { 'license' : license_url }, @@ -459,38 +462,10 @@ def add_to_sbom(proj_version_url, comp_ver_url): global bd bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) -#pprint(bd.list_resources()) - upload_sbom_file(args.spdx_file, args.project_name, args.version_name) # This will exit if it fails poll_for_upload(document.creation_info.name) -# some debug/test stubs -# TODO: delete these -#ver="https://purl-validation.saas-staging.blackduck.com/api/projects/c2b4463f-7996-4c45-8443-b69b4f82ef1d/versions/67e4f6f5-2f42-42c4-9b69-e39bad55f907" -#comp = "https://purl-validation.saas-staging.blackduck.com/api/components/fc0a76fe-70a4-4afa-9a94-c3c22d63454f/versions/fabaabb9-3b9a-4b5f-850a-39fe84c4cfc4" -#add_to_sbom(ver, comp) -#quit() - -#matchcomp, matchver = find_cust_comp("ipaddress", "1.0.23") -#if matchcomp: -# print("matched comp") -#else: -# print("no comp match") -#if matchver: -# print("matched ver") -#else: -# print("no ver match") -#comp_ver_url = create_cust_comp("MY COMPONENT z", "1", args.license_name) -# -#comp_url = "https://purl-validation.saas-staging.blackduck.com/api/components/886c04d4-28ce-4a27-be4c-f083e73a9f69" -#comp_ver_url = create_cust_comp_ver(comp_url, "701", "NOASSERTION") -# -#pv = "https://purl-validation.saas-staging.blackduck.com/api/projects/14b714d0-fa37-4684-86cc-ed4e7cc64b89/versions/b8426ca3-1e27-4045-843b-003eca72f98e" -#cv = "https://purl-validation.saas-staging.blackduck.com/api/components/886c04d4-28ce-4a27-be4c-f083e73a9f69/versions/56f64b7f-c284-457d-b593-0cf19a272a19" -#add_to_sbom(pv, cv) -#quit() - # Open unmatched component file to save name, spdxid, version, and # origin/purl for later in json format # TODO this try/except isn't quite right @@ -508,6 +483,8 @@ def add_to_sbom(proj_version_url, comp_ver_url): } projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) != 0, \ + f"Failed to locate project: {args.project_name}" assert len(projects) == 1, \ f"There should one project named {args.project_name}. Found {len(projects)}" project = projects[0] @@ -518,6 +495,8 @@ def add_to_sbom(proj_version_url, comp_ver_url): } versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name] +assert len(versions) != 0, \ + f"Failed to find project version: {args.version_name}" assert len(versions) == 1, \ f"There should be 1 version named {args.version_name}. Found {len(versions)}" version = versions[0] @@ -548,9 +527,9 @@ def add_to_sbom(proj_version_url, comp_ver_url): # Tracking unique package name + version from spdx file packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 + kb_match = None if package.external_references: foundpurl = False - kb_match = None for ref in package.external_references: # There can be multiple extrefs - try to locate a purl if (ref.reference_type == "purl"): @@ -573,7 +552,6 @@ def add_to_sbom(proj_version_url, comp_ver_url): print(f" No KB match for {package.name} {package.version}") else: nopurl += 1 - kb_match = None print(f"No pURL provided for {package.name} {package.version}") if find_comp_in_bom(matchname, matchver, version): From 1fddbdd0b86baf2b9071974df6cc8d5151e1b61e Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Mon, 28 Aug 2023 16:26:48 -0400 Subject: [PATCH 024/146] Clean up some error messages. Clarify the purpose of some variables. Handle SPDX names with space chars Fix edge case w/UNKNOWN versions --- examples/client/parse_spdx.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 9a6fb963..83541883 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -153,15 +153,15 @@ def poll_for_upload(sbom_name): sleep_time = 10 matched_scan = False + # Replace any spaces in the name with a dash to match BD + sbom_name = sbom_name.replace(' ', '-') + # TODO also check for api/projects//versions//codelocations # -- status - operationNameCode = ServerScanning, operationName=Scanning, status # -- should be COMPLETED, not IN_PROGRESS # -- operatinName: Scanning # Search for the latest scan matching our SBOM # This might be a risk for a race condition - # TODO Annoyingly, the sbom_name is not necessarily precisely our document - # name! Found a case where BD swaps a space for a "-" in the - # document name. Need to be more general in the match. params = { 'q': [f"name:{sbom_name}"], 'sort': ["updatedAt: ASC"] @@ -237,7 +237,7 @@ def upload_sbom_file(filename, project, version): try: pprint(response.json()['errorMessage']) except: - logging.error(f"Status code {response.status_code}") + logging.error(f"Status code: {response.status_code}") sys.exit(1) # Lookup the given pURL in the BD KB. @@ -282,6 +282,9 @@ def find_comp_in_bom(compname, compver, projver): if comp['componentName'].lower() != compname.lower(): # The BD API search is inexact. Force our match to be precise. continue + if compver == "UNKNOWN": + # We did not have a version specified in the first place + return True # Check component name + version name try: if comp['componentVersionName'].lower() == compver.lower(): @@ -370,15 +373,10 @@ def create_cust_comp(name, version, license): } response = bd.session.post("api/components", json=data) logging.debug(response) - if response.status_code == 412: - # Shouldn't be possible. We checked for existence earlier. - logging.error(f"Component {name} already exists") - sys.exit(1) - if response.status_code != 201: # Shouldn't be possible. We checked for existence earlier. logging.error(response.json()['errors'][0]['errorMessage']) - logging.error(f"Status code {response.status_code}") + logging.error(f"Status code: {response.status_code}") sys.exit(1) # Should be guaranteed 1 version because we just created it! @@ -427,7 +425,7 @@ def add_to_sbom(proj_version_url, comp_ver_url): response = bd.session.post(proj_version_url + "/components", json=data) if (response.status_code != 200): logging.error(response.json()['errors'][0]['errorMessage']) - logging.error(f"Status code {response.status_code}") + logging.error(f"Status code: {response.status_code}") sys.exit(1) @@ -521,9 +519,14 @@ def add_to_sbom(proj_version_url, comp_ver_url): # We hope we'll have an external reference (pURL), but we might not. extref = None purlmatch = False + # matchname/matchver can change, depending on the KB lookup step. + # These are stored separately so that we have the original names available. matchname = package.name + if package.version is None: + # Default in case one is not specified in SPDX + package.version = "UNKNOWN" matchver = package.version - print(f"Processing SPDX package: {matchname} {matchver}....") + print(f"Processing SPDX package: {matchname} version: {matchver}....") # Tracking unique package name + version from spdx file packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 From 9ecb60facb8732d17d69fb5b47ff2ac853bad2d7 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Tue, 29 Aug 2023 16:15:31 -0400 Subject: [PATCH 025/146] - add a secondary scan status function (possibly redundant) - add poll_for_sbom_scan (currently unused) - cleanup of comments - slight change in order of processing to ensure basic stuff like project/version are validated earlier - remove stray debugging code --- examples/client/parse_spdx.py | 131 +++++++++++++++++++++++++++------- 1 file changed, 105 insertions(+), 26 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 83541883..bc920fa3 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -156,18 +156,77 @@ def poll_for_upload(sbom_name): # Replace any spaces in the name with a dash to match BD sbom_name = sbom_name.replace(' ', '-') - # TODO also check for api/projects//versions//codelocations - # -- status - operationNameCode = ServerScanning, operationName=Scanning, status - # -- should be COMPLETED, not IN_PROGRESS - # -- operatinName: Scanning # Search for the latest scan matching our SBOM - # This might be a risk for a race condition params = { 'q': [f"name:{sbom_name}"], 'sort': ["updatedAt: ASC"] } - cls = bd.get_resource('codeLocations', params=params) + for cl in cls: + print(cl['name']) + # Force exact match of: spdx_doc_name + " spdx/sbom" + # BD appends the "spdx/sbom" string to the name. + if cl['name'] != sbom_name + " spdx/sbom": + continue + + matched_scan = True + for link in (cl['_meta']['links']): + # Locate the scans URL to check for status + if link['rel'] == "scans": + summaries_url = link['href'] + break + + assert(summaries_url) + params = { + 'sort': ["updatedAt: ASC"] + } + + while (max_retries): + max_retries -= 1 + for item in bd.get_items(summaries_url, params=params): + # Only checking the first item as it's the most recent + if item['scanState'] == "SUCCESS": + print("BOM scan complete") + return + elif item['scanState'] == "FAILURE": + logging.error(f"SPDX Scan Failure: {item['statusMessage']}") + sys.exit(1) + else: + # Only other state should be "STARTED" -- keep polling + print(f"Waiting for status success, currently: {item['scanState']}") + time.sleep(sleep_time) + # Break out of for loop so we always check the most recent + break + + # Handle various errors that might happen + if max_retries == 0: + logging.error("Failed to verify successful SPDX Scan in {max_retries * sleep_time} seconds") + elif not matched_scan: + logging.error(f"No scan found for SBOM: {sbom_name}") + else: + logging.error(f"Unable to verify successful scan of SBOM: {sbom_name}") + # If we got this far, it's a fatal error. + sys.exit(1) + +# Poll for successful scan of SBOM +# Inputs: +# sbom_name: Name of SBOM document (not the filename) +# version: project version to check +# Returns on success. Errors will result in fatal exit. +def poll_for_sbom_scan(sbom_name, projver): + max_retries = 30 + sleep_time = 10 + matched_scan = False + + # Replace any spaces in the name with a dash to match BD + sbom_name = sbom_name.replace(' ', '-') + + # Search for the latest scan matching our SBOM + params = { + 'q': [f"name:{sbom_name}"], + 'sort': ["updatedAt: ASC"] + } + cls = bd.get_resource('codelocations', projver, params=params) for cl in cls: # Force exact match of: spdx_doc_name + " spdx/sbom" # BD appends the "spdx/sbom" string to the name. @@ -213,7 +272,29 @@ def poll_for_upload(sbom_name): # If we got this far, it's a fatal error. sys.exit(1) -# TODO do we care about project_groups? +# Poll for BOM completion +# TODO currently unused, may delete +# Input: Name of SBOM document (not the filename, the name defined inside the json body) +# Returns on success. Errors will result in fatal exit. +def poll_for_bom_complete(proj_version_url): + max_retries = 30 + sleep_time = 10 + + while (max_retries): + max_retries -= 1 + json_data = bd.get_json(proj_version_url + "/bom-status") + if json_data['status'] == "UP_TO_DATE": + return + elif json_data['status'] == "FAILURE": + logging.error(f"BOM Scan Failed") + sys.exit(1) + elif json_data['status'] == "NOT_INCLUDED": + logging.error(f"BOM scan had no matches") + sys.exit(1) + else: + print(f"Waiting for BOM scan success, currently: {json_data['status']}") + time.sleep(sleep_time) + # Upload provided SBOM file to Black Duck # Inputs: # filename - Name of file to upload @@ -460,21 +541,6 @@ def add_to_sbom(proj_version_url, comp_ver_url): global bd bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) -upload_sbom_file(args.spdx_file, args.project_name, args.version_name) -# This will exit if it fails -poll_for_upload(document.creation_info.name) - -# Open unmatched component file to save name, spdxid, version, and -# origin/purl for later in json format -# TODO this try/except isn't quite right -try: outfile = open(args.out_file, 'w') -except: - logging.exception("Failed to open file for writing: " + args.out_file) - sys.exit(1) - -# Saved component data to write to file -comps_out = [] - # Fetch Project (can only have 1) params = { 'q': [f"name:{args.project_name}"] @@ -502,6 +568,20 @@ def add_to_sbom(proj_version_url, comp_ver_url): logging.debug(f"Found {project['name']}:{version['versionName']}") +upload_sbom_file(args.spdx_file, args.project_name, args.version_name) + +# This will exit if it fails +poll_for_upload(document.creation_info.name) +# Also exits on failure. This may be somewhat redundant. +poll_for_sbom_scan(document.creation_info.name, version) + +# Open unmatched component file to save name, spdxid, version, and +# origin/purl for later in json format +try: outfile = open(args.out_file, 'w') +except: + logging.exception("Failed to open file for writing: " + args.out_file) + sys.exit(1) + # Stats to track bom_matches = 0 kb_matches = 0 @@ -512,6 +592,8 @@ def add_to_sbom(proj_version_url, comp_ver_url): cust_ver_count = 0 # Saving all encountered components by their name+version (watching for repeats) packages = {} +# Saved component data to write to file +comps_out = [] # Walk through each component in the SPDX file for package in document.packages: @@ -535,9 +617,8 @@ def add_to_sbom(proj_version_url, comp_ver_url): foundpurl = False for ref in package.external_references: # There can be multiple extrefs - try to locate a purl + # If there should happen to be >1 purls, we only consider the first if (ref.reference_type == "purl"): - # TODO are we guaranteed only 1 purl? - # what would it mean to have >1? foundpurl = True kb_match = find_comp_in_kb(ref.locator) extref = ref.locator @@ -583,8 +664,6 @@ def add_to_sbom(proj_version_url, comp_ver_url): if kb_match: print(f" WARNING: {matchname} {matchver} in KB but not in SBOM") add_to_sbom(proj_version_url, kb_match['version']) - # temp debug to find this case - quit() # short-circuit the rest continue From 9e45f930acf1056525b75dfbd48defcdbedc1cfa Mon Sep 17 00:00:00 2001 From: Makoto Koishi Date: Wed, 30 Aug 2023 10:22:54 +0900 Subject: [PATCH 026/146] Fixed two bugs around .restconfig parameters --- examples/client/consolidated_file_report.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/client/consolidated_file_report.py b/examples/client/consolidated_file_report.py index 1a7c36b6..687b7408 100755 --- a/examples/client/consolidated_file_report.py +++ b/examples/client/consolidated_file_report.py @@ -785,9 +785,8 @@ def main(): else: bd_url = config['baseurl'] bd_token = config['api_token'] - bd_insecure = config['insecure'] - if config['debug']: - debug = 1 + bd_insecure = not config['insecure'] + debug = 1 if config['debug'] else 0 log_config(debug) From 35639bdb5f66383c28198004975cf4583f852644 Mon Sep 17 00:00:00 2001 From: Makoto Koishi Date: Wed, 30 Aug 2023 13:07:20 +0900 Subject: [PATCH 027/146] Logging level changed for exception from os.walk. Program description rephrased. --- examples/client/consolidated_file_report.py | 24 ++++++++++----------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/examples/client/consolidated_file_report.py b/examples/client/consolidated_file_report.py index 687b7408..6f575eb2 100755 --- a/examples/client/consolidated_file_report.py +++ b/examples/client/consolidated_file_report.py @@ -1,11 +1,11 @@ ''' -Created on August 23, 2023 +Created on August 30, 2023 @author: mkoishi Generate reports which consolidates information on BOM components, versions with license information, BOM files with license and copyright information from KB and license search (discoveries - file licenses or file copyrights), -and BlackDuck unmatched files. +and BlackDuck unmatched files in the target source code. Copyright (C) 2023 Synopsys, Inc. http://www.synopsys.com/ @@ -44,9 +44,10 @@ from zipfile import ZipFile program_description = \ -'''This script collects BlackDuck reports for version details and discoveries, and it generates new reports. +'''This script collects BlackDuck reports for version details and discoveries and generates new reports. The newly generated reports are a report of BOM component-versions with license information, a report of BOM files that consolidates licenses and copyright texts information from KB and discoveries (e.g., file licenses, file copyrights), and a report of BlackDuck unmatched files. -Optionally, Synopsys-Detect can be executed from the script, and if it is selected, it runs in synchronous mode. Please consider adjusting the "detect.timeout" detect parameter because completion of Synopsys-Detect is estimated to take longer. +The generated reports are found in a folder named "blackduck_consolidated_file_report" +Synopsys-Detect is executed from the script unless skip-detect option is chosen. Synopsys-Detect runs in synchronous mode and please consider adjusting the "detect.timeout" detect parameter if completion of Synopsys-Detect is estimated to take longer. Config file: API Token, hub URL and two more options need to be placed in the .restconfig.json file that must be placed in the same folder where this script resides. @@ -57,7 +58,7 @@ 2) Install PyPI modules "blackduck", "ijson" and "json2html. Examples: -1) If Synopsys-Detect is wanted to execute prior to the HTML report generation that includes file copyright texts, then +1) If Synopsys-Detect is wanted to execute prior to the HTML report generation and the report should include file-copyright texts, then python3 ./consolidated_file_report.py \ -f html -cl 2 -rr 100 \ -dp detect.blackduck.signature.scanner.snippet.matching=SNIPPET_MATCHING \ @@ -172,7 +173,7 @@ def parse_parameter(): metavar="", type=int, default=0, - help="Specify the included copyright text level. Level 0 is no copyright texts included, 1 is only copyright texts from KB included, 2 is copyright texts from KB and discoveries included.") + help="Specify the included copyright text level. Level 0 (default value) is no copyright texts included, 1 is only copyright texts from KB included, 2 is copyright texts from KB and discoveries included.") parser.add_argument("-sd", "--skip_detect", action='store_true', @@ -379,7 +380,7 @@ def get_os_path_for_unmatched(parent_dir, matched_paths): 'total_files': 0 } - log_onerror = lambda err: logging.warn(f"An error reported during folder and file traverse. {str(err)}") + log_onerror = lambda err: logging.error(f"An error reported during OS folder and file traverse. OS file report may be uncompleted.{str(err)}") for path, folder_names, file_names in os.walk(parent_dir, onerror=log_onerror): os_path_stats['total_folders'] = os_path_stats['total_folders'] + len(folder_names) os_path_stats['total_files'] = os_path_stats['total_files'] + len(file_names) @@ -699,11 +700,9 @@ def generate_file_report(hub_client, project_id, version_id, codelocations, copy for license in disc_license['licenses']: file_data['licenses'].append(license) else: - logging.debug(f"No discovery license found for file with this component: {file_data['projectName']} \ - and {file_data['projectVersion']}") + logging.debug(f"No discovery license found for file with this component: {file_data['projectName']} and {file_data['projectVersion']}") if len(disc_licenses) > 1: - logging.warning(f"More than one discovery license: {disc_licenses} found for \ - {file_data['projectName']}" and file_data['projectVersion']) + logging.warning(f"More than one discovery license: {disc_licenses} found for {file_data['projectName']}" and file_data['projectVersion']) if copyright_level != 0: origin_name = file_data['channelReleaseExternalNamespace'] + ":" + file_data['channelReleaseExternalId'] @@ -719,8 +718,7 @@ def generate_file_report(hub_client, project_id, version_id, codelocations, copy else: logging.debug(f"No discovery copyright found for file with this origin: {origin_name}") if len(disc_copyrights) > 1: - logging.warning(f"More than one discovery copyright: {disc_copyrights} found \ - for the same origin {origin_name}") + logging.warning(f"More than one discovery copyright: {disc_copyrights} found for the same origin {origin_name}") report_file_bom['bomFileEntries']['bomFiles'].append(file_data) logging.info(f"Number of the reported files {i+1}") From 023a7644b6db0fc2e55e57ebd8718ee4f391c41c Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Wed, 30 Aug 2023 10:55:42 -0400 Subject: [PATCH 028/146] nuke some TODOs and remove unused function --- examples/client/parse_spdx.py | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index bc920fa3..b873fabd 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -125,12 +125,9 @@ def spdx_validate(document): validation_messages = validate_full_spdx_document(document) print(f"SPDX validation took {time.process_time() - start} seconds") - # TODO is there a way to distinguish between something fatal and something - # BD can deal with? for validation_message in validation_messages: - # Just printing these messages intead of exiting. Later when we try to import - # the file to BD, let's plan to exit if it fails. Seeing lots of errors in the - # sample data. + # Just printing these messages intead of exiting. + # Later when the file is imported, BD errors will be fatal. logging.warning(validation_message.validation_message) # Returns MIME type to provide to scan API @@ -272,29 +269,6 @@ def poll_for_sbom_scan(sbom_name, projver): # If we got this far, it's a fatal error. sys.exit(1) -# Poll for BOM completion -# TODO currently unused, may delete -# Input: Name of SBOM document (not the filename, the name defined inside the json body) -# Returns on success. Errors will result in fatal exit. -def poll_for_bom_complete(proj_version_url): - max_retries = 30 - sleep_time = 10 - - while (max_retries): - max_retries -= 1 - json_data = bd.get_json(proj_version_url + "/bom-status") - if json_data['status'] == "UP_TO_DATE": - return - elif json_data['status'] == "FAILURE": - logging.error(f"BOM Scan Failed") - sys.exit(1) - elif json_data['status'] == "NOT_INCLUDED": - logging.error(f"BOM scan had no matches") - sys.exit(1) - else: - print(f"Waiting for BOM scan success, currently: {json_data['status']}") - time.sleep(sleep_time) - # Upload provided SBOM file to Black Duck # Inputs: # filename - Name of file to upload From aa2af4335e90d5541968b48453dbac0416508088 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Wed, 30 Aug 2023 16:37:38 -0400 Subject: [PATCH 029/146] One more round of cleanup/refactor: - Add a main() - Move parse_command_args to a function - Move proj/version lookup to a function - Truncate seconds to 2 decimal places --- examples/client/parse_spdx.py | 416 ++++++++++++++++++---------------- 1 file changed, 219 insertions(+), 197 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index b873fabd..59b5cf44 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -36,6 +36,8 @@ if the component was succesfully imported. Any missing components will be added as a custom component and then added to the Project+Verion's BOM. +All missing components are saved to a file in JSON format for future reference. + Requirements - python3 version 3.8 or newer recommended @@ -53,12 +55,11 @@ pathlib - Blackduck instance -- API token with sufficient privileges to perform project version phase - change. +- API token with sufficient privileges Install python packages with the following command: - pip3 install argparse blackduck sys logging time json spdx_tools + pip3 install argparse blackduck sys logging time json pprint pathlib spdx_tools usage: parse_spdx.py [-h] --base-url BASE_URL --token-file TOKEN_FILE --spdx-file SPDX_FILE --out-file OUT_FILE --project @@ -83,10 +84,9 @@ Version that contains the BOM components --license LICENSE_NAME License name to use for custom components (default: - NOASSERTION) + "NOASSERTION") --no-verify Disable TLS certificate verification --no-spdx-validate Disable SPDX validation - ''' from blackduck import Client @@ -103,6 +103,44 @@ from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.parse_anything import parse_file +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +# Validates BD project and version +# Inputs: +# projname - Name of project +# vername - Name of version +# Returns: Project object, Version object +def get_proj_ver(projname, vername): + # Fetch Project (can only have 1) + params = { + 'q': [f"name:{projname}"] + } + projects = [p for p in bd.get_resource('projects', params=params) + if p['name'] == projname] + assert len(projects) != 0, \ + f"Failed to locate project: {projname}" + assert len(projects) == 1, \ + f"There should one project named {projname}. Found {len(projects)}" + project = projects[0] + + # Fetch Version (can only have 1) + params = { + 'q': [f"versionName:{vername}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) + if v['versionName'] == vername] + assert len(versions) != 0, \ + f"Failed to find project version: {vername}" + assert len(versions) == 1, \ + f"There should be 1 version named {vername}. Found {len(versions)}" + version = versions[0] + + logging.debug(f"Found {project['name']}:{version['versionName']}") + return(project, version) + # Returns SPDX Document object on success, otherwise exits on parse failure # Input: file = Filename to process # Returns: SPDX document object @@ -111,7 +149,7 @@ def spdx_parse(file): start = time.process_time() try: document: Document = parse_file(file) - print(f"SPDX parsing took {time.process_time() - start} seconds") + print('SPDX parsing took {:.2f}s'.format(time.process_time() - start)) return(document) except SPDXParsingError: logging.exception("Failed to parse spdx file") @@ -123,7 +161,7 @@ def spdx_validate(document): print("Validating SPDX file...") start = time.process_time() validation_messages = validate_full_spdx_document(document) - print(f"SPDX validation took {time.process_time() - start} seconds") + print('SPDX validation took {:.2f}s'.format(time.process_time() - start)) for validation_message in validation_messages: # Just printing these messages intead of exiting. @@ -183,7 +221,7 @@ def poll_for_upload(sbom_name): for item in bd.get_items(summaries_url, params=params): # Only checking the first item as it's the most recent if item['scanState'] == "SUCCESS": - print("BOM scan complete") + print("BOM upload complete") return elif item['scanState'] == "FAILURE": logging.error(f"SPDX Scan Failure: {item['statusMessage']}") @@ -202,6 +240,7 @@ def poll_for_upload(sbom_name): logging.error(f"No scan found for SBOM: {sbom_name}") else: logging.error(f"Unable to verify successful scan of SBOM: {sbom_name}") + # If we got this far, it's a fatal error. sys.exit(1) @@ -247,7 +286,7 @@ def poll_for_sbom_scan(sbom_name, projver): for item in bd.get_items(summaries_url, params=params): # Only checking the first item as it's the most recent if item['scanState'] == "SUCCESS": - print("Scan complete") + print("BOM scan complete") return elif item['scanState'] == "FAILURE": logging.error(f"SPDX Scan Failure: {item['statusMessage']}") @@ -266,6 +305,7 @@ def poll_for_sbom_scan(sbom_name, projver): logging.error(f"No scan found for SBOM: {sbom_name}") else: logging.error(f"Unable to verify successful scan of SBOM: {sbom_name}") + # If we got this far, it's a fatal error. sys.exit(1) @@ -483,198 +523,180 @@ def add_to_sbom(proj_version_url, comp_ver_url): logging.error(f"Status code: {response.status_code}") sys.exit(1) +def parse_command_args(): + parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") + parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("--token-file", dest='token_file', required=True,help="Access token file") + parser.add_argument("--spdx-file", dest='spdx_file', required=True, help="SPDX input file") + parser.add_argument("--out-file", dest='out_file', required=True, help="Unmatched components file") + parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") + parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") + parser.add_argument("--license", dest='license_name', required=False, default="NOASSERTION", help="License name to use for custom components (default: NOASSERTION)") + parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") + parser.add_argument("--no-spdx-validate", dest='spdx_validate', action='store_false', help="Disable SPDX validation") + return parser.parse_args() + +def main(): + args = parse_command_args() + if (Path(args.spdx_file).is_file()): + document = spdx_parse(args.spdx_file) + if (args.spdx_validate): + spdx_validate(document) + else: + logging.error(f"Could not open SPDX file: {args.spdx_file}") + sys.exit(1) -parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") -parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") -parser.add_argument("--token-file", dest='token_file', required=True,help="Access token file") -parser.add_argument("--spdx-file", dest='spdx_file', required=True, help="SPDX input file") -parser.add_argument("--out-file", dest='out_file', required=True, help="Unmatched components file") -parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") -parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") -parser.add_argument("--license", dest='license_name', required=False, default="NOASSERTION", help="License name to use for custom components (default: NOASSERTION)") -parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") -parser.add_argument("--no-spdx-validate", dest='spdx_validate', action='store_false', help="Disable SPDX validation") -args = parser.parse_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() -logging.basicConfig( - level=logging.INFO, - format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" -) + global bd + bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) -if (Path(args.spdx_file).is_file()): - document = spdx_parse(args.spdx_file) - if (args.spdx_validate): - spdx_validate(document) -else: - logging.error(f"Could not open SPDX file: {args.spdx_file}") - sys.exit(1) + # Validate project/version details + project, version = get_proj_ver(args.project_name, args.version_name) + proj_version_url = version['_meta']['href'] -with open(args.token_file, 'r') as tf: - access_token = tf.readline().strip() - -global bd -bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) - -# Fetch Project (can only have 1) -params = { - 'q': [f"name:{args.project_name}"] -} -projects = [p for p in bd.get_resource('projects', params=params) - if p['name'] == args.project_name] -assert len(projects) != 0, \ - f"Failed to locate project: {args.project_name}" -assert len(projects) == 1, \ - f"There should one project named {args.project_name}. Found {len(projects)}" -project = projects[0] - -# Fetch Version (can only have 1) -params = { - 'q': [f"versionName:{args.version_name}"] -} -versions = [v for v in bd.get_resource('versions', project, params=params) - if v['versionName'] == args.version_name] -assert len(versions) != 0, \ - f"Failed to find project version: {args.version_name}" -assert len(versions) == 1, \ - f"There should be 1 version named {args.version_name}. Found {len(versions)}" -version = versions[0] -proj_version_url = version['_meta']['href'] - -logging.debug(f"Found {project['name']}:{version['versionName']}") - -upload_sbom_file(args.spdx_file, args.project_name, args.version_name) - -# This will exit if it fails -poll_for_upload(document.creation_info.name) -# Also exits on failure. This may be somewhat redundant. -poll_for_sbom_scan(document.creation_info.name, version) - -# Open unmatched component file to save name, spdxid, version, and -# origin/purl for later in json format -try: outfile = open(args.out_file, 'w') -except: - logging.exception("Failed to open file for writing: " + args.out_file) - sys.exit(1) + # Upload the provided SBOM + upload_sbom_file(args.spdx_file, args.project_name, args.version_name) -# Stats to track -bom_matches = 0 -kb_matches = 0 -nopurl = 0 -nomatch = 0 -package_count = 0 -cust_comp_count = 0 -cust_ver_count = 0 -# Saving all encountered components by their name+version (watching for repeats) -packages = {} -# Saved component data to write to file -comps_out = [] - -# Walk through each component in the SPDX file -for package in document.packages: - package_count += 1 - # We hope we'll have an external reference (pURL), but we might not. - extref = None - purlmatch = False - # matchname/matchver can change, depending on the KB lookup step. - # These are stored separately so that we have the original names available. - matchname = package.name - if package.version is None: - # Default in case one is not specified in SPDX - package.version = "UNKNOWN" - matchver = package.version - print(f"Processing SPDX package: {matchname} version: {matchver}....") - # Tracking unique package name + version from spdx file - packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 - - kb_match = None - if package.external_references: - foundpurl = False - for ref in package.external_references: - # There can be multiple extrefs - try to locate a purl - # If there should happen to be >1 purls, we only consider the first - if (ref.reference_type == "purl"): - foundpurl = True - kb_match = find_comp_in_kb(ref.locator) - extref = ref.locator - break - if not foundpurl: + # Wait for scan completeion. Will exit if it fails. + poll_for_upload(document.creation_info.name) + # Also exits on failure. This may be somewhat redundant. + poll_for_sbom_scan(document.creation_info.name, version) + + # Open unmatched component file to save name, spdxid, version, and + # origin/purl for later in json format + try: outfile = open(args.out_file, 'w') + except: + logging.exception("Failed to open file for writing: " + args.out_file) + sys.exit(1) + + # Stats to track + bom_matches = 0 + kb_matches = 0 + nopurl = 0 + nomatch = 0 + package_count = 0 + cust_comp_count = 0 + cust_ver_count = 0 + # Saving all encountered components by their name+version + # Used for debugging repeated package data + packages = {} + # Saved component data to write to file + comps_out = [] + + # Walk through each component in the SPDX file + for package in document.packages: + package_count += 1 + # We hope we'll have an external reference (pURL), but we might not. + extref = None + purlmatch = False + # matchname/matchver can change, depending on the KB lookup step. + # These are stored separately to keep the original names handy + matchname = package.name + if package.version is None: + # Default in case one is not specified in SPDX + package.version = "UNKNOWN" + matchver = package.version + print(f"Processing SPDX package: {matchname} version: {matchver}....") + # Tracking unique package name + version combos from spdx file + packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 + + kb_match = None + if package.external_references: + foundpurl = False + for ref in package.external_references: + # There can be multiple extrefs - try to locate a purl + # If there should happen to be multiple purls, + # we only consider the first. + if (ref.reference_type == "purl"): + foundpurl = True + kb_match = find_comp_in_kb(ref.locator) + extref = ref.locator + break + if not foundpurl: + nopurl += 1 + print(f" No pURL provided for {package.name} {package.version}") + if (kb_match): + # Update package name and version to reflect the KB name/ver + print(f" KB match for {package.name} {package.version}") + kb_matches+=1 + matchname = kb_match['componentName'] + matchver = kb_match['versionName'] + else: + print(f" No KB match for {package.name} {package.version}") + else: nopurl += 1 - print(f" No pURL provided for {package.name} {package.version}") - if (kb_match): - # Update package name and version to reflect the KB name/ver - print(f" KB match for {package.name} {package.version}") - kb_matches+=1 - matchname = kb_match['componentName'] - matchver = kb_match['versionName'] + print(f" No pURL provided for {package.name} {package.version}") + + if find_comp_in_bom(matchname, matchver, version): + bom_matches += 1 + print(f" Found component in BOM: {matchname} {matchver}") + # It's in the BOM so we are happy + # Everything else below is related to adding to the BOM + continue + + # If we've gotten this far, the package is not in the BOM. + # Now we need to figure out: + # - Is it already in the KB and we need to add it? (should be rare) + # - Do we need to add a custom component? + # - Do we need to add a version to an existing custom component? + nomatch += 1 + print(f" Not present in BOM: {matchname} {matchver}") + comp_data = { + "name": package.name, + "spdx_id": package.spdx_id, + "version": package.version, + "origin": extref + } + comps_out.append(comp_data) + + # KB match was successful, but it wasn't in the BOM for some reason + if kb_match: + print(f" WARNING: {matchname} {matchver} in KB but not in SBOM") + add_to_sbom(proj_version_url, kb_match['version']) + # short-circuit the rest + continue + + # Check if custom component already exists + comp_url, comp_ver_url = find_cust_comp(package.name, package.version) + + if not comp_url: + # Custom component did not exist, so create it + cust_comp_count += 1 + comp_ver_url = create_cust_comp(package.name, package.version, + args.license_name) + elif comp_url and not comp_ver_url: + # Custom component existed, but not the version we care about + cust_ver_count += 1 + print(f" Adding version {package.version} to custom component {package.name}") + comp_ver_url = create_cust_comp_ver(comp_url, package.version, \ + args.license_name) else: - print(f" No KB match for {package.name} {package.version}") - else: - nopurl += 1 - print(f"No pURL provided for {package.name} {package.version}") - - if find_comp_in_bom(matchname, matchver, version): - bom_matches += 1 - print(f" Found component in BOM: {matchname} {matchver}") - # It's in the BOM so we are happy - # Everything else below is related to adding to the BOM - continue - - # If we've gotten this far, the package is not in the BOM. - # Now we need to figure out: - # - Is it already in the KB and we need to add it? (should be rare) - # - Do we need to add a custom component? - # - Do we need to add a version to an existing custom component? - nomatch += 1 - print(f" Not present in BOM: {matchname} {matchver}") - comp_data = { - "name": package.name, - "spdx_id": package.spdx_id, - "version": package.version, - "origin": extref - } - comps_out.append(comp_data) - - # KB match was successful, but it wasn't in the BOM for some reason - if kb_match: - print(f" WARNING: {matchname} {matchver} in KB but not in SBOM") - add_to_sbom(proj_version_url, kb_match['version']) - # short-circuit the rest - continue - - # Check if custom component already exists - comp_url, comp_ver_url = find_cust_comp(package.name, package.version) - - if not comp_url: - # Custom component did not exist, so create it - cust_comp_count += 1 - comp_ver_url = create_cust_comp(package.name, package.version, - args.license_name) - elif comp_url and not comp_ver_url: - # Custom component existed, but not the version we care about - cust_ver_count += 1 - print(f" Adding version {package.version} to custom component {package.name}") - comp_ver_url = create_cust_comp_ver(comp_url, package.version, args.license_name) - else: - print(" Custom component already exists, not in SBOM") - - # Shouldn't be possible - assert(comp_ver_url), f"No component URL found for {package.name} {package.version}" - - print(f" Adding component to SBOM: {package.name} aka {matchname} {package.version}") - add_to_sbom(proj_version_url, comp_ver_url) - -# Save unmatched components -json.dump(comps_out, outfile) -outfile.close() - -print("\nStats: ") -print("------") -print(f" SPDX packages processed: {package_count}") -print(f" Packages missing from BOM: {nomatch}") -print(f" BOM matches: {bom_matches}") -print(f" KB matches: {kb_matches}") -print(f" Packages missing purl: {nopurl}") -print(f" Custom components created: {cust_comp_count}") -print(f" Custom component versions created: {cust_ver_count}") -#pprint(packages) -print(f" {len(packages)} unique packages processed") + print(" Custom component already exists, not in SBOM") + + # Shouldn't be possible + assert(comp_ver_url), f"No component URL found for {package.name} {package.version}" + + print(f" Adding component to SBOM: {package.name} aka {matchname} {package.version}") + add_to_sbom(proj_version_url, comp_ver_url) + + # Save unmatched components + json.dump(comps_out, outfile) + outfile.close() + + print("\nStats: ") + print("------") + print(f" SPDX packages processed: {package_count}") + print(f" Packages missing from BOM: {nomatch}") + print(f" BOM matches: {bom_matches}") + print(f" KB matches: {kb_matches}") + print(f" Packages missing purl: {nopurl}") + print(f" Custom components created: {cust_comp_count}") + print(f" Custom component versions created: {cust_ver_count}") + #for debugging + #pprint(packages) + print(f" {len(packages)} unique packages processed") + +if __name__ == "__main__": + sys.exit(main()) From abfc7e6dbdd3f3427168f11edf4105f220bef310 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Fri, 1 Sep 2023 09:42:36 -0400 Subject: [PATCH 030/146] Bugfixes from testing: - handle a package name w/empty string (print warning/skip them) - handle empty string version name (use UNKNOWN) - trim leading/trailing whitespace chars from package name/version strings - minor comment fixup and removal of stray debugging --- examples/client/parse_spdx.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 59b5cf44..1dd00fe4 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -149,12 +149,13 @@ def spdx_parse(file): start = time.process_time() try: document: Document = parse_file(file) - print('SPDX parsing took {:.2f}s'.format(time.process_time() - start)) - return(document) except SPDXParsingError: logging.exception("Failed to parse spdx file") sys.exit(1) + print('SPDX parsing took {:.2f}s'.format(time.process_time() - start)) + return(document) + # Validates the SPDX file. Logs all validation messages as warnings. # Input: SPDX document object def spdx_validate(document): @@ -198,7 +199,6 @@ def poll_for_upload(sbom_name): } cls = bd.get_resource('codeLocations', params=params) for cl in cls: - print(cl['name']) # Force exact match of: spdx_doc_name + " spdx/sbom" # BD appends the "spdx/sbom" string to the name. if cl['name'] != sbom_name + " spdx/sbom": @@ -559,7 +559,7 @@ def main(): # Upload the provided SBOM upload_sbom_file(args.spdx_file, args.project_name, args.version_name) - # Wait for scan completeion. Will exit if it fails. + # Wait for scan completion. Will exit if it fails. poll_for_upload(document.creation_info.name) # Also exits on failure. This may be somewhat redundant. poll_for_sbom_scan(document.creation_info.name, version) @@ -591,14 +591,24 @@ def main(): # We hope we'll have an external reference (pURL), but we might not. extref = None purlmatch = False + + if package.name == "": + # Strange case where the package name is empty. Skip it. + logging.warning("WARNING: package name empty, skipping") + continue + # Trim any odd leading/trailing space or newlines + package.name = package.name.strip() + # matchname/matchver can change, depending on the KB lookup step. # These are stored separately to keep the original names handy matchname = package.name - if package.version is None: + if package.version is None or package.version == "": # Default in case one is not specified in SPDX package.version = "UNKNOWN" + package.version = package.version.strip() matchver = package.version - print(f"Processing SPDX package: {matchname} version: {matchver}....") + print(f"Processing SPDX package: {matchname} version: {matchver}...") + # Tracking unique package name + version combos from spdx file packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 @@ -643,6 +653,8 @@ def main(): # - Do we need to add a version to an existing custom component? nomatch += 1 print(f" Not present in BOM: {matchname} {matchver}") + + # Missing component data to write to a file for reference comp_data = { "name": package.name, "spdx_id": package.spdx_id, From 418762047007d605e69d004238e9087b1314e4ac Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Tue, 5 Sep 2023 16:23:53 -0400 Subject: [PATCH 031/146] checkpoint, saving improved scan detection code work in progress --- examples/client/parse_spdx.py | 205 +++++++++++++++++----------------- 1 file changed, 104 insertions(+), 101 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 1dd00fe4..64c5a6f3 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -103,6 +103,12 @@ from spdx_tools.spdx.parser.error import SPDXParsingError from spdx_tools.spdx.parser.parse_anything import parse_file +# Used when we are polling for successful upload and processing +global MAX_RETRIES +global SLEEP +MAX_RETRIES = 30 +SLEEP = 5 + logging.basicConfig( level=logging.INFO, format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" @@ -181,88 +187,60 @@ def get_sbom_mime_type(filename): return 'application/spdx' return None -# Poll for successful scan of SBOM. -# Input: Name of SBOM document (not the filename, the name defined inside the json body) -# Returns on success. Errors will result in fatal exit. -def poll_for_upload(sbom_name): - max_retries = 30 - sleep_time = 10 - matched_scan = False +def poll_notifications_for_success(cl, proj_version_url, summaries_url): + # We want to locate a notification for + # VERSION_BOM_CODE_LOCATION_BOM_COMPUTED + # matching our proj_version_url and our codelocation + retries = MAX_RETRIES + sleep_time = SLEEP - # Replace any spaces in the name with a dash to match BD - sbom_name = sbom_name.replace(' ', '-') + # current theory: if a scan happened and we matched NOTHING, we + # aren't going to get a BOM_COMPUTED notification. so is there any type + # of notif that we DO get? - # Search for the latest scan matching our SBOM params = { - 'q': [f"name:{sbom_name}"], - 'sort': ["updatedAt: ASC"] + 'filter': ["notificationType:VERSION_BOM_CODE_LOCATION_BOM_COMPUTED"], + 'sort' : ["createdAt: ASC"] } - cls = bd.get_resource('codeLocations', params=params) - for cl in cls: - # Force exact match of: spdx_doc_name + " spdx/sbom" - # BD appends the "spdx/sbom" string to the name. - if cl['name'] != sbom_name + " spdx/sbom": - continue - - matched_scan = True - for link in (cl['_meta']['links']): - # Locate the scans URL to check for status - if link['rel'] == "scans": - summaries_url = link['href'] - break - - assert(summaries_url) - params = { - 'sort': ["updatedAt: ASC"] - } - - while (max_retries): - max_retries -= 1 - for item in bd.get_items(summaries_url, params=params): - # Only checking the first item as it's the most recent - if item['scanState'] == "SUCCESS": - print("BOM upload complete") - return - elif item['scanState'] == "FAILURE": - logging.error(f"SPDX Scan Failure: {item['statusMessage']}") - sys.exit(1) - else: - # Only other state should be "STARTED" -- keep polling - print(f"Waiting for status success, currently: {item['scanState']}") - time.sleep(sleep_time) - # Break out of for loop so we always check the most recent - break - - # Handle various errors that might happen - if max_retries == 0: - logging.error("Failed to verify successful SPDX Scan in {max_retries * sleep_time} seconds") - elif not matched_scan: - logging.error(f"No scan found for SBOM: {sbom_name}") - else: - logging.error(f"Unable to verify successful scan of SBOM: {sbom_name}") - # If we got this far, it's a fatal error. + while (retries): + retries -= 1 + for result in bd.get_items("/api/notifications", params=params): + if 'projectVersion' not in result['content']: + # skip it (shouldn't be possible due to the filter) + continue + # We're checking the entire list of notifications, but ours is + # likely to be the first. Walking the whole list to make + # sure we find an exact match. + if result['content']['projectVersion'] == proj_version_url and \ + result['content']['codeLocation'] == cl['_meta']['href'] and \ + result['content']['scanSummary'] == summaries_url: + print("BOM calculation complete") + return + + print("Waiting for BOM calculation to complete") + time.sleep(sleep_time) + + logging.error(f"Failed to verify successful BOM computed in {retries * sleep_time} seconds") sys.exit(1) -# Poll for successful scan of SBOM -# Inputs: -# sbom_name: Name of SBOM document (not the filename) -# version: project version to check +# Poll for successful scan of SBOM. +# Input: Name of SBOM document (not the filename, the name defined inside the json body) # Returns on success. Errors will result in fatal exit. -def poll_for_sbom_scan(sbom_name, projver): - max_retries = 30 - sleep_time = 10 +def poll_for_sbom_complete(sbom_name, proj_version_url): + retries = MAX_RETRIES + sleep_time = SLEEP matched_scan = False # Replace any spaces in the name with a dash to match BD sbom_name = sbom_name.replace(' ', '-') - # Search for the latest scan matching our SBOM + # Search for the latest scan matching our SBOM name params = { 'q': [f"name:{sbom_name}"], 'sort': ["updatedAt: ASC"] } - cls = bd.get_resource('codelocations', projver, params=params) + cls = bd.get_resource('codeLocations', params=params) for cl in cls: # Force exact match of: spdx_doc_name + " spdx/sbom" # BD appends the "spdx/sbom" string to the name. @@ -272,42 +250,67 @@ def poll_for_sbom_scan(sbom_name, projver): matched_scan = True for link in (cl['_meta']['links']): # Locate the scans URL to check for status - if link['rel'] == "scans": - summaries_url = link['href'] + if link['rel'] == "latest-scan": + latest_url = link['href'] break - assert(summaries_url) - params = { - 'sort': ["updatedAt: ASC"] - } + assert latest_url, "Failed to locate latest-scan reference" + if not matched_scan: + logging.error(f"No scan found for SBOM: {sbom_name}") + sys.exit(1) - while (max_retries): - max_retries -= 1 - for item in bd.get_items(summaries_url, params=params): - # Only checking the first item as it's the most recent - if item['scanState'] == "SUCCESS": - print("BOM scan complete") - return - elif item['scanState'] == "FAILURE": - logging.error(f"SPDX Scan Failure: {item['statusMessage']}") - sys.exit(1) - else: - # Only other state should be "STARTED" -- keep polling - print(f"Waiting for status success, currently: {item['scanState']}") - time.sleep(sleep_time) - # Break out of for loop so we always check the most recent - break + # Wait for scanState = SUCCESS + while (retries): + json_data = bd.get_json(latest_url) + retries -= 1 + if json_data['scanState'] == "SUCCESS": + print("BOM upload complete") + break + elif json_data['scanState'] == "FAILURE": + logging.error(f"SPDX Scan Failure: {json_data['statusMessage']}") + sys.exit(1) + else: + # Only other state should be "STARTED" -- keep polling + print(f"Waiting for status success, currently: {json_data['scanState']}") + time.sleep(sleep_time) + + # If there were ZERO matches, there will never be a notification of + # BOM import success. Short-circuit that check and treat this as success. + if json_data['matchCount'] == 0: + print("No KB matches in BOM, continuing...") + return + + # Save the codelocation summaries_url + summaries_url = json_data['_meta']['href'] + + # Greedy match - extract the scan id out of the URL + #scanid = re.findall(r'.*\/(.*)', json_data['_meta']['href']) + # proj_Version_url/bom-status/scanid does NOT WORK + + # TODO this seems actually fairly pointless - it get stuck in UP_TO_DATE + retries = MAX_RETRIES + while (retries): + json_data = bd.get_json(proj_version_url + "/bom-status") + retries -= 1 + if json_data['status'] == "UP_TO_DATE": + print("BOM import complete") + break + elif json_data['status'] == "FAILURE": + logging.error(f"BOM Import failure: {json_data['status']}") + sys.exit(1) + else: + print(f"Waiting for BOM import completion, current status: {json_data['status']}") + time.sleep(sleep_time) - # Handle various errors that might happen - if max_retries == 0: - logging.error("Failed to verify successful SPDX Scan in {max_retries * sleep_time} seconds") - elif not matched_scan: - logging.error(f"No scan found for SBOM: {sbom_name}") - else: - logging.error(f"Unable to verify successful scan of SBOM: {sbom_name}") + if retries == 0: + logging.error("Failed to verify successful SBOM import in {retries * sleep_time} seconds") + sys.exit(1) - # If we got this far, it's a fatal error. - sys.exit(1) + # Finally check notifications + poll_notifications_for_success(cl, proj_version_url, summaries_url) + + # Any errors above already resulted in fatal exit + return # Upload provided SBOM file to Black Duck # Inputs: @@ -328,7 +331,7 @@ def upload_sbom_file(filename, project, version): logging.error(f"File {filename} is already mapped to a different project version") if response.status_code != 201: - logging.error(f"Failed to upload SPDX file:") + logging.error(f"Failed to upload SPDX file") try: pprint(response.json()['errorMessage']) except: @@ -560,9 +563,7 @@ def main(): upload_sbom_file(args.spdx_file, args.project_name, args.version_name) # Wait for scan completion. Will exit if it fails. - poll_for_upload(document.creation_info.name) - # Also exits on failure. This may be somewhat redundant. - poll_for_sbom_scan(document.creation_info.name, version) + poll_for_sbom_complete(document.creation_info.name, proj_version_url) # Open unmatched component file to save name, spdxid, version, and # origin/purl for later in json format @@ -630,7 +631,7 @@ def main(): if (kb_match): # Update package name and version to reflect the KB name/ver print(f" KB match for {package.name} {package.version}") - kb_matches+=1 + kb_matches += 1 matchname = kb_match['componentName'] matchver = kb_match['versionName'] else: @@ -667,6 +668,8 @@ def main(): if kb_match: print(f" WARNING: {matchname} {matchver} in KB but not in SBOM") add_to_sbom(proj_version_url, kb_match['version']) + # TODO TEMP DEBUG TO CATCH THIS + quit() # short-circuit the rest continue From 2af04082941d135fcf8f47b2e0e43e625cbfad12 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Wed, 6 Sep 2023 16:42:57 -0400 Subject: [PATCH 032/146] Improve detection of scan/BOM success. Clean up a few comments. Print package details for the case where a package name was empty. --- examples/client/parse_spdx.py | 83 +++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 64c5a6f3..335066bb 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -107,13 +107,15 @@ global MAX_RETRIES global SLEEP MAX_RETRIES = 30 -SLEEP = 5 +SLEEP = 10 logging.basicConfig( level=logging.INFO, format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" ) +logging.getLogger("blackduck").setLevel(logging.CRITICAL) + # Validates BD project and version # Inputs: # projname - Name of project @@ -187,17 +189,18 @@ def get_sbom_mime_type(filename): return 'application/spdx' return None -def poll_notifications_for_success(cl, proj_version_url, summaries_url): - # We want to locate a notification for - # VERSION_BOM_CODE_LOCATION_BOM_COMPUTED - # matching our proj_version_url and our codelocation +# Poll for notification alerting us of successful BOM computation +# +# Inputs: +# cl_url - Code Loction URL to match +# proj_version_url - Project Version URL to match +# summaries_url - Summaries URL from codelocation +# +# Returns on success. Errors are fatal. +def poll_notifications_for_success(cl_url, proj_version_url, summaries_url): retries = MAX_RETRIES sleep_time = SLEEP - # current theory: if a scan happened and we matched NOTHING, we - # aren't going to get a BOM_COMPUTED notification. so is there any type - # of notif that we DO get? - params = { 'filter': ["notificationType:VERSION_BOM_CODE_LOCATION_BOM_COMPUTED"], 'sort' : ["createdAt: ASC"] @@ -207,13 +210,12 @@ def poll_notifications_for_success(cl, proj_version_url, summaries_url): retries -= 1 for result in bd.get_items("/api/notifications", params=params): if 'projectVersion' not in result['content']: - # skip it (shouldn't be possible due to the filter) + # Shouldn't be possible due to the filter continue - # We're checking the entire list of notifications, but ours is - # likely to be the first. Walking the whole list to make - # sure we find an exact match. + # We're checking the entire list of notifications, but ours should + # be near the top. if result['content']['projectVersion'] == proj_version_url and \ - result['content']['codeLocation'] == cl['_meta']['href'] and \ + result['content']['codeLocation'] == cl_url and \ result['content']['scanSummary'] == summaries_url: print("BOM calculation complete") return @@ -221,16 +223,22 @@ def poll_notifications_for_success(cl, proj_version_url, summaries_url): print("Waiting for BOM calculation to complete") time.sleep(sleep_time) - logging.error(f"Failed to verify successful BOM computed in {retries * sleep_time} seconds") + logging.error(f"Failed to verify successful BOM computed in {MAX_RETRIES * sleep_time} seconds") sys.exit(1) # Poll for successful scan of SBOM. -# Input: Name of SBOM document (not the filename, the name defined inside the json body) +# Inputs: +# +# sbom_name: Name of SBOM document (not the filename, the name defined +# inside the json body) +# proj_version_url: Project version url +# # Returns on success. Errors will result in fatal exit. def poll_for_sbom_complete(sbom_name, proj_version_url): retries = MAX_RETRIES sleep_time = SLEEP matched_scan = False + cl_url = None # Replace any spaces in the name with a dash to match BD sbom_name = sbom_name.replace(' ', '-') @@ -242,23 +250,29 @@ def poll_for_sbom_complete(sbom_name, proj_version_url): } cls = bd.get_resource('codeLocations', params=params) for cl in cls: + if matched_scan: + break # Force exact match of: spdx_doc_name + " spdx/sbom" # BD appends the "spdx/sbom" string to the name. if cl['name'] != sbom_name + " spdx/sbom": continue matched_scan = True + cl_url = cl['_meta']['href'] + for link in (cl['_meta']['links']): # Locate the scans URL to check for status if link['rel'] == "latest-scan": latest_url = link['href'] break - assert latest_url, "Failed to locate latest-scan reference" if not matched_scan: logging.error(f"No scan found for SBOM: {sbom_name}") sys.exit(1) + assert latest_url, "Failed to locate latest-scan reference" + assert cl_url, "Failed to locate codelocation reference" + # Wait for scanState = SUCCESS while (retries): json_data = bd.get_json(latest_url) @@ -271,23 +285,19 @@ def poll_for_sbom_complete(sbom_name, proj_version_url): sys.exit(1) else: # Only other state should be "STARTED" -- keep polling - print(f"Waiting for status success, currently: {json_data['scanState']}") + print(f"Waiting for scan completion, currently: {json_data['scanState']}") time.sleep(sleep_time) # If there were ZERO matches, there will never be a notification of - # BOM import success. Short-circuit that check and treat this as success. + # BOM import success. Short-circuit the check and treat this as success. if json_data['matchCount'] == 0: - print("No KB matches in BOM, continuing...") + print("No BOM KB matches, continuing...") return # Save the codelocation summaries_url summaries_url = json_data['_meta']['href'] - # Greedy match - extract the scan id out of the URL - #scanid = re.findall(r'.*\/(.*)', json_data['_meta']['href']) - # proj_Version_url/bom-status/scanid does NOT WORK - - # TODO this seems actually fairly pointless - it get stuck in UP_TO_DATE + # Check the bom-status endpoint for success retries = MAX_RETRIES while (retries): json_data = bd.get_json(proj_version_url + "/bom-status") @@ -295,19 +305,20 @@ def poll_for_sbom_complete(sbom_name, proj_version_url): if json_data['status'] == "UP_TO_DATE": print("BOM import complete") break - elif json_data['status'] == "FAILURE": - logging.error(f"BOM Import failure: {json_data['status']}") + elif json_data['status'] == "UP_TO_DATE_WITH_ERRORS" or \ + json_data['status'] == "PROCESSING_WITH_ERRORS": + logging.error(f"BOM Import failure: status is {json_data['status']}") sys.exit(1) else: print(f"Waiting for BOM import completion, current status: {json_data['status']}") time.sleep(sleep_time) if retries == 0: - logging.error("Failed to verify successful SBOM import in {retries * sleep_time} seconds") + logging.error(f"Failed to verify successful SBOM import in {retries * sleep_time} seconds") sys.exit(1) # Finally check notifications - poll_notifications_for_success(cl, proj_version_url, summaries_url) + poll_notifications_for_success(cl_url, proj_version_url, summaries_url) # Any errors above already resulted in fatal exit return @@ -580,8 +591,7 @@ def main(): package_count = 0 cust_comp_count = 0 cust_ver_count = 0 - # Saving all encountered components by their name+version - # Used for debugging repeated package data + # Used for tracking repeated package data packages = {} # Saved component data to write to file comps_out = [] @@ -595,8 +605,10 @@ def main(): if package.name == "": # Strange case where the package name is empty. Skip it. - logging.warning("WARNING: package name empty, skipping") + logging.warning("WARNING: Skipping empty package name. Package info:") + pprint(package) continue + # Trim any odd leading/trailing space or newlines package.name = package.name.strip() @@ -617,9 +629,8 @@ def main(): if package.external_references: foundpurl = False for ref in package.external_references: - # There can be multiple extrefs - try to locate a purl - # If there should happen to be multiple purls, - # we only consider the first. + # There can be multiple extrefs; try to locate a purl. + # If there are multiple purls, use the first one. if (ref.reference_type == "purl"): foundpurl = True kb_match = find_comp_in_kb(ref.locator) @@ -668,8 +679,6 @@ def main(): if kb_match: print(f" WARNING: {matchname} {matchver} in KB but not in SBOM") add_to_sbom(proj_version_url, kb_match['version']) - # TODO TEMP DEBUG TO CATCH THIS - quit() # short-circuit the rest continue From 3cad5d61e08107a229981f470395273b3a376944 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Thu, 7 Sep 2023 14:28:55 -0400 Subject: [PATCH 033/146] Add more graceful HTTP response checks --- examples/client/parse_spdx.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 335066bb..6dcbedbc 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -341,7 +341,7 @@ def upload_sbom_file(filename, project, version): if response.status_code == 409: logging.error(f"File {filename} is already mapped to a different project version") - if response.status_code != 201: + if not response.ok: logging.error(f"Failed to upload SPDX file") try: pprint(response.json()['errorMessage']) @@ -482,7 +482,7 @@ def create_cust_comp(name, version, license): } response = bd.session.post("api/components", json=data) logging.debug(response) - if response.status_code != 201: + if not response.ok: # Shouldn't be possible. We checked for existence earlier. logging.error(response.json()['errors'][0]['errorMessage']) logging.error(f"Status code: {response.status_code}") @@ -516,7 +516,7 @@ def create_cust_comp_ver(comp_url, version, license): logging.error(f"Version {version} already exists for component") sys.exit(1) - if response.status_code != 201: + if not response.ok: logging.error(f"Failed to add Version {version} to component") sys.exit(1) @@ -532,7 +532,7 @@ def add_to_sbom(proj_version_url, comp_ver_url): 'component': comp_ver_url } response = bd.session.post(proj_version_url + "/components", json=data) - if (response.status_code != 200): + if not response.ok: logging.error(response.json()['errors'][0]['errorMessage']) logging.error(f"Status code: {response.status_code}") sys.exit(1) From 856989644aa10311559a1fdc21fa8835b0e620d5 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Mon, 11 Sep 2023 14:53:32 -0400 Subject: [PATCH 034/146] skip blackduck-version extrefs --- examples/client/parse_spdx.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 6dcbedbc..5bdb9749 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -626,16 +626,21 @@ def main(): packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 kb_match = None + bd_proj = False if package.external_references: foundpurl = False for ref in package.external_references: - # There can be multiple extrefs; try to locate a purl. - # If there are multiple purls, use the first one. + # There can be multiple extrefs; try to locate a pURL. + # If there are multiple pURLs, use the first one. if (ref.reference_type == "purl"): foundpurl = True kb_match = find_comp_in_kb(ref.locator) extref = ref.locator break + # Skip BD project/versions. These occur in BD-generated BOMs. + if (ref.reference_type == "BlackDuck-Version"): + bd_proj = True + break if not foundpurl: nopurl += 1 print(f" No pURL provided for {package.name} {package.version}") @@ -648,14 +653,17 @@ def main(): else: print(f" No KB match for {package.name} {package.version}") else: + # No external references field was provided nopurl += 1 print(f" No pURL provided for {package.name} {package.version}") + if bd_proj: + print(f" Skipping BD project/version in BOM: {package.name} {package.version}") + continue + if find_comp_in_bom(matchname, matchver, version): bom_matches += 1 print(f" Found component in BOM: {matchname} {matchver}") - # It's in the BOM so we are happy - # Everything else below is related to adding to the BOM continue # If we've gotten this far, the package is not in the BOM. From 00cc4210892c165b00ed890d69acabf1df213795 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Tue, 12 Sep 2023 16:35:56 -0400 Subject: [PATCH 035/146] updates to reflect API URL and parameter renaming --- examples/client/parse_spdx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 5bdb9749..b4c1aaff 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -359,9 +359,9 @@ def upload_sbom_file(filename, project, version): # No match: None def find_comp_in_kb(extref): params = { - 'packageUrl': extref + 'purl': extref } - for result in bd.get_items("/api/search/purl-components", params=params): + for result in bd.get_items("/api/search/kb-purl-component", params=params): # Should be exactly 1 match when successful return(result) From 618235682fa77a1dd0c07e500ea7208f7b5f99f2 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Thu, 21 Sep 2023 15:10:35 -0400 Subject: [PATCH 036/146] make this modular: - rename main to import_sbom - make a stub function to handle command-line args so it can still be called standalone - make some args optional to import_sbom and handle their absence --- examples/client/parse_spdx.py | 65 ++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index b4c1aaff..5b2a7e0e 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -550,38 +550,60 @@ def parse_command_args(): parser.add_argument("--no-spdx-validate", dest='spdx_validate', action='store_false', help="Disable SPDX validation") return parser.parse_args() -def main(): +# Stub to support invocation as a standalone script +# Parses the command-line args, creates a BD object, and inokes import_sbom +def spdx_main_parse_args(): args = parse_command_args() - if (Path(args.spdx_file).is_file()): - document = spdx_parse(args.spdx_file) - if (args.spdx_validate): - spdx_validate(document) - else: - logging.error(f"Could not open SPDX file: {args.spdx_file}") - sys.exit(1) - with open(args.token_file, 'r') as tf: access_token = tf.readline().strip() + bdobj = Client(base_url=args.base_url, token=access_token, verify=args.verify) + import_sbom(bdobj, args.project_name, args.version_name, args.spdx_file, \ + args.out_file, args.license_name, args.spdx_validate) + +# Main entry point +# +# Inputs: +# bdobj - BD Client Object +# projname - Name of project +# vername - Name of version +# spdxfile - SPDX file location +# outfile (Optional) - Name of file to write missing component data to in JSON. +# Default: No file written +# license_name - Name of license to use for custom components +# Default: NOASSERTION +# do_spdx_validate - Validate the SPDX file? (Boolean) +# Default: True +def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ + license_name="NOASSERTION", do_spdx_validate=True): global bd - bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) + bd = bdobj + + if (Path(spdxfile).is_file()): + document = spdx_parse(spdxfile) + if (do_spdx_validate): + spdx_validate(document) + else: + logging.error(f"Could not open SPDX file: {spdxfile}") + sys.exit(1) # Validate project/version details - project, version = get_proj_ver(args.project_name, args.version_name) + project, version = get_proj_ver(projname, vername) proj_version_url = version['_meta']['href'] # Upload the provided SBOM - upload_sbom_file(args.spdx_file, args.project_name, args.version_name) + upload_sbom_file(spdxfile, projname, vername) # Wait for scan completion. Will exit if it fails. poll_for_sbom_complete(document.creation_info.name, proj_version_url) # Open unmatched component file to save name, spdxid, version, and # origin/purl for later in json format - try: outfile = open(args.out_file, 'w') - except: - logging.exception("Failed to open file for writing: " + args.out_file) - sys.exit(1) + if outfile: + try: outfile = open(outfile, 'w') + except: + logging.exception("Failed to open file for writing: " + outfile) + sys.exit(1) # Stats to track bom_matches = 0 @@ -697,13 +719,13 @@ def main(): # Custom component did not exist, so create it cust_comp_count += 1 comp_ver_url = create_cust_comp(package.name, package.version, - args.license_name) + license_name) elif comp_url and not comp_ver_url: # Custom component existed, but not the version we care about cust_ver_count += 1 print(f" Adding version {package.version} to custom component {package.name}") comp_ver_url = create_cust_comp_ver(comp_url, package.version, \ - args.license_name) + license_name) else: print(" Custom component already exists, not in SBOM") @@ -714,8 +736,9 @@ def main(): add_to_sbom(proj_version_url, comp_ver_url) # Save unmatched components - json.dump(comps_out, outfile) - outfile.close() + if outfile: + json.dump(comps_out, outfile) + outfile.close() print("\nStats: ") print("------") @@ -731,4 +754,4 @@ def main(): print(f" {len(packages)} unique packages processed") if __name__ == "__main__": - sys.exit(main()) + sys.exit(spdx_main_parse_args()) From 2f6a8f9a5ec3be5ccc6ef0d7a1f247a8e9a185f3 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Tue, 26 Sep 2023 11:12:00 -0400 Subject: [PATCH 037/146] gracefully handle failure to locate scan info: - add try/retry loop - print more status messages - print detailed debug data on failure - add some initializations and assertions for safety --- examples/client/parse_spdx.py | 54 ++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 5b2a7e0e..4b996906 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -238,6 +238,7 @@ def poll_for_sbom_complete(sbom_name, proj_version_url): retries = MAX_RETRIES sleep_time = SLEEP matched_scan = False + latest_url = None cl_url = None # Replace any spaces in the name with a dash to match BD @@ -248,32 +249,56 @@ def poll_for_sbom_complete(sbom_name, proj_version_url): 'q': [f"name:{sbom_name}"], 'sort': ["updatedAt: ASC"] } - cls = bd.get_resource('codeLocations', params=params) - for cl in cls: + + while (retries): + cls = bd.get_resource('codeLocations', params=params) + retries -= 1 if matched_scan: + # Exit the while() break - # Force exact match of: spdx_doc_name + " spdx/sbom" - # BD appends the "spdx/sbom" string to the name. - if cl['name'] != sbom_name + " spdx/sbom": - continue - - matched_scan = True - cl_url = cl['_meta']['href'] - - for link in (cl['_meta']['links']): - # Locate the scans URL to check for status - if link['rel'] == "latest-scan": - latest_url = link['href'] + # Save the CL data as we go for debugging + backupcls = [] + for cl in cls: + backupcls.append(cl) + if matched_scan: + # Exit the inner for() break + print(f"Searching scans for {sbom_name}...") + # Force exact match of: spdx_doc_name + " spdx/sbom" + # BD appends the "spdx/sbom" string to the name. + if cl['name'] != sbom_name + " spdx/sbom": + # No match, keep searching + print(f" {cl['name']} != {sbom_name}" + " spdx/sbom") + continue + + print(" Scan located") + matched_scan = True + cl_url = cl['_meta']['href'] + + print("Checking for latest-scan info...") + for link in (cl['_meta']['links']): + # Locate the scans URL to check for status + if link['rel'] == "latest-scan": + print(" Located latest-scan") + latest_url = link['href'] + break + + # We walked the list of code locations and didn't find a match + if not matched_scan: + print(f" Waiting to locate scan...") + time.sleep(sleep_time) if not matched_scan: logging.error(f"No scan found for SBOM: {sbom_name}") + print("\nCodelocations API data:\n") + pprint(backupcls) sys.exit(1) assert latest_url, "Failed to locate latest-scan reference" assert cl_url, "Failed to locate codelocation reference" # Wait for scanState = SUCCESS + retries = MAX_RETRIES while (retries): json_data = bd.get_json(latest_url) retries -= 1 @@ -288,6 +313,7 @@ def poll_for_sbom_complete(sbom_name, proj_version_url): print(f"Waiting for scan completion, currently: {json_data['scanState']}") time.sleep(sleep_time) + assert json_data, "Failed to locate scanState data" # If there were ZERO matches, there will never be a notification of # BOM import success. Short-circuit the check and treat this as success. if json_data['matchCount'] == 0: From a8896e0fd750ae5b9398eef77623b813a8064851 Mon Sep 17 00:00:00 2001 From: Makoto Date: Thu, 28 Sep 2023 16:55:39 +0900 Subject: [PATCH 038/146] Added link to snippet ui in the header --- examples/client/consolidated_file_report.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/examples/client/consolidated_file_report.py b/examples/client/consolidated_file_report.py index 6f575eb2..f014799d 100755 --- a/examples/client/consolidated_file_report.py +++ b/examples/client/consolidated_file_report.py @@ -89,6 +89,7 @@ BLACKDUCK_REPORT_MEDIATYPE = "application/vnd.blackducksoftware.report-4+json" blackduck_report_download_api = "/api/projects/{projectId}/versions/{projectVersionId}/reports/{reportId}/download" blackduck_link_component_ui_api = "/api/projects/{projectId}/versions/{projectVersionId}/components" +blackduck_link_snippet_ui_api = "/api/projects/{projectId}/versions/{projectVersionId}/source-trees" # BD version details report blackduck_create_version_report_api = "/api/versions/{projectVersionId}/reports" blackduck_version_report_filename = "./blackduck_version_report_for_{projectVersionId}.zip" @@ -105,6 +106,7 @@ REPORT_COMPONENT_BOM = "/component_bom_report" REPORT_FILE_BOM = "/file_bom_report" REPORT_DISCOVERY = "/discovery" +BLACKDUCK_SNIPPET_FILTER = "?filter=bomMatchType%3Asnippet&offset=0&limit=100" # Retries to wait for BD report creation. RETRY_LIMIT can be overwritten by the script parameter. RETRY_LIMIT = 30 RETRY_TIMER = 30 @@ -115,7 +117,8 @@ 'detectParameters': [], 'scanDateTime': '', 'blackDuckVersion': '', - 'linkToBlackDuckProjectVersionInUI': '' + 'linkToBlackDuckProjectVersionInUI': '', + 'linkToBlackDuckSnippetMatchInUI': '' }, 'fileInventory': { 'linkToUnmatchedOsFileData': "", @@ -632,6 +635,9 @@ def generate_file_report(hub_client, project_id, version_id, codelocations, copy blackduck_link_component_ui_api.replace("{projectId}", project_id).replace("{projectVersionId}", version_id) report_content['configurationSettings']['linkToBlackDuckProjectVersionInUI'] = \ hub_client.base_url + blackduck_link_component_ui_api.replace("{projectId}", project_id).replace("{projectVersionId}", version_id) + report_content['configurationSettings']['linkToBlackDuckSnippetMatchInUI'] = \ + hub_client.base_url + blackduck_link_snippet_ui_api.replace("{projectId}", project_id).replace("{projectVersionId}", version_id) \ + + BLACKDUCK_SNIPPET_FILTER # Report body - Component BOM, file BOM with Discoveries data version_report_zip = get_version_detail_report(hub_client, project_id, version_id, retries) From 744f4c7dd93cda53461e55d727bf07ff1da83336 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 13 Oct 2023 10:34:53 -0400 Subject: [PATCH 039/146] crypto to custom field example --- examples/client/crypto-to-custom.py | 84 +++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 examples/client/crypto-to-custom.py diff --git a/examples/client/crypto-to-custom.py b/examples/client/crypto-to-custom.py new file mode 100644 index 00000000..d701feaa --- /dev/null +++ b/examples/client/crypto-to-custom.py @@ -0,0 +1,84 @@ +import argparse +from blackduck import Client +from pprint import pprint +import logging +import sys +import json + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.INFO) +logging.getLogger("urllib3").setLevel(logging.INFO) +logging.getLogger("blackduck").setLevel(logging.INFO) + +def find_project_by_name(project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + +def parse_command_args(): + + parser = argparse.ArgumentParser("product-from-bom.py") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-pn", "--project-name", required=True, help="Project Name") + parser.add_argument("-vn", "--version-name", required=True, help="Version Name") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("--reset", action='store_true') + return parser.parse_args() + +def set_custom_field(field, url, value): + payload = {"fields": [{"customField": field['_meta']['href'],"values": value}]} + headers = {"Accept": "application/vnd.blackducksoftware.bill-of-materials-6+json", + "Content-Type": "application/vnd.blackducksoftware.bill-of-materials-6+json"} + response = bd.session.put(url, data=json.dumps(payload), headers=headers) + print(response) + +def process_project_version(args): + project = find_project_by_name(args.project_name) + version = find_project_version_by_name(project, args.version_name) + + components = bd.get_resource('components',version) + for component in components: + print (component['componentName'], component['componentVersionName']) + custom_fields = bd.get_resource('custom-fields',component, items=False) + custom_fields_url = custom_fields['_meta']['href'] + c = [x for x in custom_fields['items'] if x['label'] == 'BadCrypto'][0] + resources = bd.list_resources(component) + if 'crypto-algorithms' in resources.keys(): + crypto_algorithms = bd.get_resource('crypto-algorithms', component) + for crypto in crypto_algorithms: + if crypto['knownWeaknesses']: + pprint('Has Weakness') + value = ['true'] + if args.reset: + value = [] + set_custom_field(c, custom_fields_url, value=value) + break + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + global bd + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + + process_project_version(args) + +if __name__ == "__main__": + sys.exit(main()) + From a154cd77fa23331038cc74f76a855735fa762de3 Mon Sep 17 00:00:00 2001 From: mkumykov <38922450+mkumykov@users.noreply.github.com> Date: Fri, 13 Oct 2023 10:58:41 -0400 Subject: [PATCH 040/146] Update crypto-to-custom.py --- examples/client/crypto-to-custom.py | 79 ++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/examples/client/crypto-to-custom.py b/examples/client/crypto-to-custom.py index d701feaa..b7dd54cc 100644 --- a/examples/client/crypto-to-custom.py +++ b/examples/client/crypto-to-custom.py @@ -1,3 +1,78 @@ +''' +Created on October 12, 2023 +@author: kumykov + +Copyright (C) 2023 Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +This script is provided as an example of populating custom field data +based on BOM components crypto information. +The goal is to enable policy functionality that would be triggered by +cryptographic features of a component. + +The scriot will analyze ciphers included in a component and will set +a BOM Component custom field value to reflec that. + +Requirements + +- python3 version 3.8 or newer recommended +- the following packages are used by the script and should be installed + prior to use: + argparse + blackduck + logging + sys + json + pprint +- Blackduck instance +- API token with sufficient privileges to perform project version phase + change. + +Install python packages with the following command: + + pip3 install argparse blackduck logging sys json pprint + +Using + +Script expects a boolean custom field labeled "BadCrypto" on a BOM Component. +A policy that is triggered by BadCrypto custom field value used to visualise +results. + +usage: crypto-to-custom.py [-h] -u BASE_URL -t TOKEN_FILE -pn PROJECT_NAME -vn VERSION_NAME [-nv] [--reset] + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -pn PROJECT_NAME, --project-name PROJECT_NAME + Project Name + -vn VERSION_NAME, --version-name VERSION_NAME + Version Name + -nv, --no-verify Disable TLS certificate verification + --reset Undo the changes made by thjis script + + +''' + import argparse from blackduck import Client from pprint import pprint @@ -32,13 +107,13 @@ def find_project_version_by_name(project, version_name): def parse_command_args(): - parser = argparse.ArgumentParser("product-from-bom.py") + parser = argparse.ArgumentParser("crypto-to-custom.py") parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") parser.add_argument("-t", "--token-file", required=True, help="File containing access token") parser.add_argument("-pn", "--project-name", required=True, help="Project Name") parser.add_argument("-vn", "--version-name", required=True, help="Version Name") parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") - parser.add_argument("--reset", action='store_true') + parser.add_argument("--reset", action='store_true', help="Undo the changes made by thjis script") return parser.parse_args() def set_custom_field(field, url, value): From 7648feb9d71fccf4871c28d81a2e6ed41cc232c5 Mon Sep 17 00:00:00 2001 From: mkumykov <38922450+mkumykov@users.noreply.github.com> Date: Fri, 13 Oct 2023 11:02:25 -0400 Subject: [PATCH 041/146] Update crypto-to-custom.py --- examples/client/crypto-to-custom.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/client/crypto-to-custom.py b/examples/client/crypto-to-custom.py index b7dd54cc..96e3e9e8 100644 --- a/examples/client/crypto-to-custom.py +++ b/examples/client/crypto-to-custom.py @@ -27,8 +27,8 @@ The goal is to enable policy functionality that would be triggered by cryptographic features of a component. -The scriot will analyze ciphers included in a component and will set -a BOM Component custom field value to reflec that. +The script will analyze ciphers included in a component and will set +a BOM Component custom field value to reflect that a known weakness is present. Requirements @@ -52,7 +52,7 @@ Using Script expects a boolean custom field labeled "BadCrypto" on a BOM Component. -A policy that is triggered by BadCrypto custom field value used to visualise +A policy that is triggered by BadCrypto custom field value used to visualize results. usage: crypto-to-custom.py [-h] -u BASE_URL -t TOKEN_FILE -pn PROJECT_NAME -vn VERSION_NAME [-nv] [--reset] From 31e54e60ad7006cca392ffe45a3c01fa1d96105d Mon Sep 17 00:00:00 2001 From: mkumykov <38922450+mkumykov@users.noreply.github.com> Date: Fri, 13 Oct 2023 11:03:57 -0400 Subject: [PATCH 042/146] Update crypto-to-custom.py --- examples/client/crypto-to-custom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/client/crypto-to-custom.py b/examples/client/crypto-to-custom.py index 96e3e9e8..1ecf6ecd 100644 --- a/examples/client/crypto-to-custom.py +++ b/examples/client/crypto-to-custom.py @@ -68,7 +68,7 @@ -vn VERSION_NAME, --version-name VERSION_NAME Version Name -nv, --no-verify Disable TLS certificate verification - --reset Undo the changes made by thjis script + --reset Undo the changes made by this script ''' @@ -113,7 +113,7 @@ def parse_command_args(): parser.add_argument("-pn", "--project-name", required=True, help="Project Name") parser.add_argument("-vn", "--version-name", required=True, help="Version Name") parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") - parser.add_argument("--reset", action='store_true', help="Undo the changes made by thjis script") + parser.add_argument("--reset", action='store_true', help="Undo the changes made by this script") return parser.parse_args() def set_custom_field(field, url, value): From 68291b16b74afe9f044d23935fb8ee1a0813b86d Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Fri, 13 Oct 2023 12:03:06 -0400 Subject: [PATCH 043/146] Add KB match on BD IDs - Avoids adding unnecessary custom components - Various comment fixup - Removed stray unused variable - Make the stats a little more clear --- examples/client/parse_spdx.py | 124 ++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 35 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 4b996906..b773bc90 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -2,13 +2,6 @@ Created on August 15, 2023 @author: swright -##################### DISCLAIMER ########################## -## This script was created for a specific purpose and ## -## SHOULD NOT BE USED as a general purpose utility. ## -## For general purpose utility use ## -## /examples/client/generate_sbom.py ## -########################################################### - Copyright (C) 2023 Synopsys, Inc. http://www.blackducksoftware.com/ @@ -38,6 +31,10 @@ All missing components are saved to a file in JSON format for future reference. +Version History +1.0 2023-09-26 Initial Release +1.1 2023-10-13 Updates to improve component matching of BD Component IDs + Requirements - python3 version 3.8 or newer recommended @@ -394,6 +391,39 @@ def find_comp_in_kb(extref): # Fall through -- lookup failed return(None) +# Lookup the given BD Compononent Version in the BD KB. +# Note: Match source will be one of: KB, CUSTOM, or KB_MODIFIED +# Any of these should be should be acceptable +# +# Inputs: +# Component UUID +# Component Version UUID +# +# Returns: +# kb_match dictionary that mimics the format returned by find_comp_in_kb: +# keys are: componentName, versionName, version (url of component version) +# No match returns None +def find_comp_id_in_kb(comp, ver): + kb_match = {} + try: + json_data = bd.get_json(f"/api/components/{comp}") + except: + # No component match + return None + kb_match['componentName'] = json_data['name'] + + try: + json_data = bd.get_json(f"/api/components/{comp}/versions/{ver}") + except: + # No component version match + return None + kb_match['versionName'] = json_data['versionName'] + + # Add the url of the component-version + kb_match['version'] = json_data['_meta']['href'] + + return kb_match + # Locate component name + version in BOM # Inputs: # compname - Component name to locate @@ -586,6 +616,16 @@ def spdx_main_parse_args(): import_sbom(bdobj, args.project_name, args.version_name, args.spdx_file, \ args.out_file, args.license_name, args.spdx_validate) +# Normalize a BD UUID or URL in the extrefs section to be consistently formatted +# Input: Black Duck component or version ID string from SPDX file +# Output: UUID +def normalize_id(id): + # Strip trailing '/' + id = id.rstrip('/') + # Ensure only the UUID remains + id = id.split('/')[-1] + return id + # Main entry point # # Inputs: @@ -635,7 +675,9 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ bom_matches = 0 kb_matches = 0 nopurl = 0 - nomatch = 0 + not_in_bom = 0 + cust_added_to_bom = 0 + kb_match_added_to_bom = 0 package_count = 0 cust_comp_count = 0 cust_ver_count = 0 @@ -647,9 +689,9 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ # Walk through each component in the SPDX file for package in document.packages: package_count += 1 - # We hope we'll have an external reference (pURL), but we might not. + # We hope we'll have an external reference (pURL or KBID), but it + # is possible to have neither. extref = None - purlmatch = False if package.name == "": # Strange case where the package name is empty. Skip it. @@ -671,27 +713,37 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ print(f"Processing SPDX package: {matchname} version: {matchver}...") # Tracking unique package name + version combos from spdx file + # This is only used for debugging and stats purposes packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 kb_match = None - bd_proj = False if package.external_references: - foundpurl = False + # Build dictionary of extrefs for easy access + extrefs = {} for ref in package.external_references: - # There can be multiple extrefs; try to locate a pURL. - # If there are multiple pURLs, use the first one. - if (ref.reference_type == "purl"): - foundpurl = True - kb_match = find_comp_in_kb(ref.locator) - extref = ref.locator - break + # Older BD release prepend this string; strip it + reftype = ref.reference_type.lstrip("LocationRef-") + extrefs[reftype] = ref.locator + + if "purl" in extrefs: + # purl is the preferred lookup + kb_match = find_comp_in_kb(ref.locator) + extref = ref.locator + elif "BlackDuck-Component" in extrefs: + compid = normalize_id(extrefs['BlackDuck-Component']) + verid = normalize_id(extrefs['BlackDuck-ComponentVersion']) + + # Lookup by KB ID + kb_match = find_comp_id_in_kb(compid, verid) + extref = extrefs['BlackDuck-Component'] + elif "BlackDuck-Version" in extrefs: # Skip BD project/versions. These occur in BD-generated BOMs. - if (ref.reference_type == "BlackDuck-Version"): - bd_proj = True - break - if not foundpurl: + print(f" Skipping BD project/version in BOM: {package.name} {package.version}") + continue + else: nopurl += 1 - print(f" No pURL provided for {package.name} {package.version}") + print(f" No pURL or KB ID provided for {package.name} {package.version}") + if (kb_match): # Update package name and version to reflect the KB name/ver print(f" KB match for {package.name} {package.version}") @@ -701,14 +753,10 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ else: print(f" No KB match for {package.name} {package.version}") else: - # No external references field was provided + # No external references field was provide nopurl += 1 print(f" No pURL provided for {package.name} {package.version}") - if bd_proj: - print(f" Skipping BD project/version in BOM: {package.name} {package.version}") - continue - if find_comp_in_bom(matchname, matchver, version): bom_matches += 1 print(f" Found component in BOM: {matchname} {matchver}") @@ -716,10 +764,10 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ # If we've gotten this far, the package is not in the BOM. # Now we need to figure out: - # - Is it already in the KB and we need to add it? (should be rare) + # - Is it already in the KB and we need to add it? # - Do we need to add a custom component? # - Do we need to add a version to an existing custom component? - nomatch += 1 + not_in_bom += 1 print(f" Not present in BOM: {matchname} {matchver}") # Missing component data to write to a file for reference @@ -733,7 +781,9 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ # KB match was successful, but it wasn't in the BOM for some reason if kb_match: - print(f" WARNING: {matchname} {matchver} in KB but not in SBOM") + kb_match_added_to_bom += 1 + print(f" WARNING: {matchname} {matchver} found in KB but not in SBOM - adding it") + # kb_match['version'] contains the url of the component-version to add add_to_sbom(proj_version_url, kb_match['version']) # short-circuit the rest continue @@ -759,6 +809,7 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ assert(comp_ver_url), f"No component URL found for {package.name} {package.version}" print(f" Adding component to SBOM: {package.name} aka {matchname} {package.version}") + cust_added_to_bom += 1 add_to_sbom(proj_version_url, comp_ver_url) # Save unmatched components @@ -769,15 +820,18 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ print("\nStats: ") print("------") print(f" SPDX packages processed: {package_count}") - print(f" Packages missing from BOM: {nomatch}") + # package_count above could have repeated packages in it + print(f" Unique packages processed: {len(packages)}") + print(f" Packages missing purl or KBID: {nopurl}") print(f" BOM matches: {bom_matches}") print(f" KB matches: {kb_matches}") - print(f" Packages missing purl: {nopurl}") print(f" Custom components created: {cust_comp_count}") print(f" Custom component versions created: {cust_ver_count}") + print(f" Packages missing from BOM: {not_in_bom}") + print(f" Custom components added to BOM: {cust_added_to_bom}") + print(f" KB matches added to BOM: {kb_match_added_to_bom}") #for debugging #pprint(packages) - print(f" {len(packages)} unique packages processed") if __name__ == "__main__": sys.exit(spdx_main_parse_args()) From 5a4b1054c2549c54e56ce5ef01e3c647001a9260 Mon Sep 17 00:00:00 2001 From: swright-synopsys Date: Fri, 3 Nov 2023 14:24:48 -0500 Subject: [PATCH 044/146] - Handle BD component with no version - Fix bug related to extrefs with both purl and BD component data - Check if project every had a "non-SBOM" scan and exit if so - Fix some invalid sort parmaeter formatting - Limit notification checking to last 24 hours --- examples/client/parse_spdx.py | 68 +++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index b773bc90..1563d3be 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -34,6 +34,11 @@ Version History 1.0 2023-09-26 Initial Release 1.1 2023-10-13 Updates to improve component matching of BD Component IDs +1.2 2023-11-03 - Handle BD component with no version + - Fix bug related to extrefs with both purl and BD component data + - Check if project every had a "non-SBOM" scan and exit if so + - Fix some invalid sort parmaeter formatting + - Limit notification checking to last 24 hours Requirements @@ -50,13 +55,14 @@ spdx_tools re pathlib + datetime - Blackduck instance - API token with sufficient privileges Install python packages with the following command: - pip3 install argparse blackduck sys logging time json pprint pathlib spdx_tools + pip3 install datetime argparse blackduck sys logging time json pprint pathlib spdx_tools usage: parse_spdx.py [-h] --base-url BASE_URL --token-file TOKEN_FILE --spdx-file SPDX_FILE --out-file OUT_FILE --project @@ -92,6 +98,7 @@ import logging import time import json +from datetime import datetime,timedelta,timezone import re from pprint import pprint from pathlib import Path @@ -103,7 +110,7 @@ # Used when we are polling for successful upload and processing global MAX_RETRIES global SLEEP -MAX_RETRIES = 30 +MAX_RETRIES = 60 SLEEP = 10 logging.basicConfig( @@ -198,9 +205,15 @@ def poll_notifications_for_success(cl_url, proj_version_url, summaries_url): retries = MAX_RETRIES sleep_time = SLEEP + # Limit the query to the last 24 hours (very conservative but also + # keeps us from having to walk thousands of notifications every time) + today=datetime.now().astimezone(timezone.utc) + yesterday=today - timedelta(days=1) + start=yesterday.strftime("%Y-%m-%dT%H:%M:%S.000Z") params = { 'filter': ["notificationType:VERSION_BOM_CODE_LOCATION_BOM_COMPUTED"], - 'sort' : ["createdAt: ASC"] + 'sort' : ["createdAt DESC"], + 'startDate' : [start] } while (retries): @@ -211,18 +224,33 @@ def poll_notifications_for_success(cl_url, proj_version_url, summaries_url): continue # We're checking the entire list of notifications, but ours should # be near the top. - if result['content']['projectVersion'] == proj_version_url and \ - result['content']['codeLocation'] == cl_url and \ - result['content']['scanSummary'] == summaries_url: - print("BOM calculation complete") - return + if (result['content']['projectVersion'] == proj_version_url and + result['content']['codeLocation'] == cl_url and + result['content']['scanSummary'] == summaries_url): + print("BOM calculation complete") + return print("Waiting for BOM calculation to complete") + # For debugging + #print(f"Searching Notifications for:\n Proj_version: {proj_version_url}\n" + + # f" CodeLocation: {cl_url}\n Summaries: {summaries_url}") time.sleep(sleep_time) logging.error(f"Failed to verify successful BOM computed in {MAX_RETRIES * sleep_time} seconds") sys.exit(1) +# Check if this project-version ever had a non-SBOM scan +# If so, we do not want to step on any toes and exit the script before +# attempting any SBOM import. +# Note: Uses an internal API for simplicity +def check_for_existing_scan(projver): + headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} + for source in bd.get_items(f"{projver}/source-trees", headers=headers): + if not re.fullmatch(r".+spdx/sbom$", source['name']): + logging.error(f"Project has a non-SBOM scan. Details:") + pprint(source) + sys.exit(1) + # Poll for successful scan of SBOM. # Inputs: # @@ -244,7 +272,7 @@ def poll_for_sbom_complete(sbom_name, proj_version_url): # Search for the latest scan matching our SBOM name params = { 'q': [f"name:{sbom_name}"], - 'sort': ["updatedAt: ASC"] + 'sort': ["updatedAt DESC"] } while (retries): @@ -411,6 +439,14 @@ def find_comp_id_in_kb(comp, ver): # No component match return None kb_match['componentName'] = json_data['name'] + if ver is None: + # Special case where a component was provided but no version. + # Stick the component URL in the version field which we will later use + # to update the BOM. The name of this field is now overloaded but + # reusing it to stay generic. + kb_match['version'] = json_data['_meta']['href'] + kb_match['versionName'] = "UNKNOWN" + return kb_match try: json_data = bd.get_json(f"/api/components/{comp}/versions/{ver}") @@ -656,6 +692,7 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ # Validate project/version details project, version = get_proj_ver(projname, vername) proj_version_url = version['_meta']['href'] + check_for_existing_scan(proj_version_url) # Upload the provided SBOM upload_sbom_file(spdxfile, projname, vername) @@ -727,12 +764,15 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ if "purl" in extrefs: # purl is the preferred lookup - kb_match = find_comp_in_kb(ref.locator) - extref = ref.locator + kb_match = find_comp_in_kb(extrefs['purl']) + extref = extrefs['purl'] elif "BlackDuck-Component" in extrefs: compid = normalize_id(extrefs['BlackDuck-Component']) - verid = normalize_id(extrefs['BlackDuck-ComponentVersion']) - + try: + verid = normalize_id(extrefs['BlackDuck-ComponentVersion']) + except: + print(" BD Component specified with no version") + verid = None # Lookup by KB ID kb_match = find_comp_id_in_kb(compid, verid) extref = extrefs['BlackDuck-Component'] @@ -783,7 +823,7 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ if kb_match: kb_match_added_to_bom += 1 print(f" WARNING: {matchname} {matchver} found in KB but not in SBOM - adding it") - # kb_match['version'] contains the url of the component-version to add + # kb_match['version'] contains the component url to add add_to_sbom(proj_version_url, kb_match['version']) # short-circuit the rest continue From 7942d1592a649b5f8e3ab320f6008601ac92a348 Mon Sep 17 00:00:00 2001 From: swright-synopsys Date: Fri, 3 Nov 2023 14:26:01 -0500 Subject: [PATCH 045/146] typo fix --- examples/client/parse_spdx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 1563d3be..735c0f3c 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -37,7 +37,7 @@ 1.2 2023-11-03 - Handle BD component with no version - Fix bug related to extrefs with both purl and BD component data - Check if project every had a "non-SBOM" scan and exit if so - - Fix some invalid sort parmaeter formatting + - Fix some invalid sort parameter formatting - Limit notification checking to last 24 hours Requirements From 10fc4dce10f1a86cc9badf277a47684c847f9873 Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Tue, 14 Nov 2023 17:02:17 -0500 Subject: [PATCH 046/146] several bug fixes related to counting and component accuracy: - Force encoding utf8 when opening file - Use the component URL from API call in the version query - resolves some situations where the KB version lookup fails - Update find_comp_in_bom to return the matching URL instead of True/False - Track unique BOM matches by tracking the matched component URL returned by find_comp_in_com - Track the count of skipped items from the SPDX - Make the unique package tracking more accurate - do not include skipped items - Create fall-through matching. First check BD component, then the purl info (rather than only checking the purl) --- examples/client/parse_spdx.py | 65 ++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 735c0f3c..5746519d 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -39,9 +39,19 @@ - Check if project every had a "non-SBOM" scan and exit if so - Fix some invalid sort parameter formatting - Limit notification checking to last 24 hours +1.3 2023-11-14 - Force encoding utf8 when opening file + - Use the component URL from API call in the version query - resolves + some situations where the KB version lookup fails + - Update find_comp_in_bom to return the matching URL instead of + True/False + - Track unique BOM matches by tracking the matched component URL + returned by find_comp_in_com + - Track the count of skipped items from the SPDX + - Make the unique package tracking more accurate - do not include skipped items + - Create fall-through matching. First check BD component, then the purl info + (rather than only checking the purl) Requirements - - python3 version 3.8 or newer recommended - The following packages are used by the script and should be installed prior to use: @@ -184,7 +194,7 @@ def spdx_validate(document): # Returns MIME type to provide to scan API # Input: filename to check def get_sbom_mime_type(filename): - with open(filename, 'r') as f: + with open(filename, 'r', encoding="utf8") as f: data = f.readlines() content = " ".join(data) if 'CycloneDX' in content: @@ -448,8 +458,10 @@ def find_comp_id_in_kb(comp, ver): kb_match['versionName'] = "UNKNOWN" return kb_match + # Update the component url to match the one returned by API + comp_url = json_data['_meta']['href'] try: - json_data = bd.get_json(f"/api/components/{comp}/versions/{ver}") + json_data = bd.get_json(f"{comp_url}/versions/{ver}") except: # No component version match return None @@ -466,7 +478,7 @@ def find_comp_id_in_kb(comp, ver): # compver - Component version to locate # projver - Project version to locate component in BOM # -# Returns: True on success, False on failure +# Returns: Component match URL on success, None on failure def find_comp_in_bom(compname, compver, projver): have_match = False num_match = 0 @@ -485,16 +497,16 @@ def find_comp_in_bom(compname, compver, projver): continue if compver == "UNKNOWN": # We did not have a version specified in the first place - return True + return comp['component'] # Check component name + version name try: if comp['componentVersionName'].lower() == compver.lower(): - return True + return comp['componentVersion'] except: # Handle situation where it's missing the version name for some reason print(f"comp {compname} in BOM has no version!") - return False - return False + return None + return None # Verifies if a custom component and version already exist in the system. # @@ -718,8 +730,11 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ package_count = 0 cust_comp_count = 0 cust_ver_count = 0 - # Used for tracking repeated package data + skip_count = 0 + # Used for tracking repeated package data, not including skips packages = {} + # Used for tracking unique BOM matches + bom_packages = {} # Saved component data to write to file comps_out = [] @@ -733,6 +748,7 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ if package.name == "": # Strange case where the package name is empty. Skip it. logging.warning("WARNING: Skipping empty package name. Package info:") + skip_count += 1 pprint(package) continue @@ -749,10 +765,6 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ matchver = package.version print(f"Processing SPDX package: {matchname} version: {matchver}...") - # Tracking unique package name + version combos from spdx file - # This is only used for debugging and stats purposes - packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 - kb_match = None if package.external_references: # Build dictionary of extrefs for easy access @@ -762,11 +774,8 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ reftype = ref.reference_type.lstrip("LocationRef-") extrefs[reftype] = ref.locator - if "purl" in extrefs: - # purl is the preferred lookup - kb_match = find_comp_in_kb(extrefs['purl']) - extref = extrefs['purl'] - elif "BlackDuck-Component" in extrefs: + if "BlackDuck-Component" in extrefs: + # Prefer BD component lookup if available compid = normalize_id(extrefs['BlackDuck-Component']) try: verid = normalize_id(extrefs['BlackDuck-ComponentVersion']) @@ -776,8 +785,18 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ # Lookup by KB ID kb_match = find_comp_id_in_kb(compid, verid) extref = extrefs['BlackDuck-Component'] + if not kb_match: + # BD comp lookup failed, so try purl instead + if "purl" in extrefs: + kb_match = find_comp_in_kb(extrefs['purl']) + extref = extrefs['purl'] + elif "purl" in extrefs: + # If no BD component details are available + kb_match = find_comp_in_kb(extrefs['purl']) + extref = extrefs['purl'] elif "BlackDuck-Version" in extrefs: # Skip BD project/versions. These occur in BD-generated BOMs. + skip_count += 1 print(f" Skipping BD project/version in BOM: {package.name} {package.version}") continue else: @@ -797,7 +816,10 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ nopurl += 1 print(f" No pURL provided for {package.name} {package.version}") - if find_comp_in_bom(matchname, matchver, version): + bom_comp = find_comp_in_bom(matchname, matchver, version) + if bom_comp: + bom_packages[bom_comp] = bom_packages.get(bom_comp, 0) + 1 + packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 bom_matches += 1 print(f" Found component in BOM: {matchname} {matchver}") continue @@ -809,6 +831,7 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ # - Do we need to add a version to an existing custom component? not_in_bom += 1 print(f" Not present in BOM: {matchname} {matchver}") + packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 # Missing component data to write to a file for reference comp_data = { @@ -862,16 +885,18 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ print(f" SPDX packages processed: {package_count}") # package_count above could have repeated packages in it print(f" Unique packages processed: {len(packages)}") + print(f" Skipped: {skip_count}") print(f" Packages missing purl or KBID: {nopurl}") print(f" BOM matches: {bom_matches}") + print(f" Unique BOM matches: {len(bom_packages)}") print(f" KB matches: {kb_matches}") print(f" Custom components created: {cust_comp_count}") print(f" Custom component versions created: {cust_ver_count}") print(f" Packages missing from BOM: {not_in_bom}") print(f" Custom components added to BOM: {cust_added_to_bom}") print(f" KB matches added to BOM: {kb_match_added_to_bom}") - #for debugging #pprint(packages) + #pprint(bom_packages) if __name__ == "__main__": sys.exit(spdx_main_parse_args()) From 08ef01dd4bc8c2dc30afc0367101e6664638f0cc Mon Sep 17 00:00:00 2001 From: Shane Wright Date: Tue, 21 Nov 2023 13:45:54 -0500 Subject: [PATCH 047/146] Improve accuracy of BOM component lookup - Search the import-events - If that search fails, fall through to the standard component search - Track unique BOM matches by the name+ver string instead of URL, as import-events only gives us the matched name/ver data - Some minor typo fixes --- examples/client/parse_spdx.py | 78 +++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py index 5746519d..c20c01e7 100644 --- a/examples/client/parse_spdx.py +++ b/examples/client/parse_spdx.py @@ -45,11 +45,13 @@ - Update find_comp_in_bom to return the matching URL instead of True/False - Track unique BOM matches by tracking the matched component URL - returned by find_comp_in_com + returned by find_comp_in_bom - Track the count of skipped items from the SPDX - Make the unique package tracking more accurate - do not include skipped items - Create fall-through matching. First check BD component, then the purl info (rather than only checking the purl) +1.4 2023-11-21 - Check the component-import-events API for improved BOM + component searching accuracy Requirements - python3 version 3.8 or newer recommended @@ -268,7 +270,8 @@ def check_for_existing_scan(projver): # inside the json body) # proj_version_url: Project version url # -# Returns on success. Errors will result in fatal exit. +# Returns summaries_url on success (used for processing the import events later) +# Errors will result in fatal exit. def poll_for_sbom_complete(sbom_name, proj_version_url): retries = MAX_RETRIES sleep_time = SLEEP @@ -382,7 +385,7 @@ def poll_for_sbom_complete(sbom_name, proj_version_url): poll_notifications_for_success(cl_url, proj_version_url, summaries_url) # Any errors above already resulted in fatal exit - return + return summaries_url # Upload provided SBOM file to Black Duck # Inputs: @@ -472,13 +475,21 @@ def find_comp_id_in_kb(comp, ver): return kb_match +# Locate component name + version in component-import-events +# Returns matched name+version on success, None on failure +def find_comp_import_events(match_dict, compname, compver): + key = compname+compver + if key in match_dict: + return match_dict[key] + return None + # Locate component name + version in BOM # Inputs: # compname - Component name to locate # compver - Component version to locate # projver - Project version to locate component in BOM # -# Returns: Component match URL on success, None on failure +# Returns: Component name+version string on success, None on failure def find_comp_in_bom(compname, compver, projver): have_match = False num_match = 0 @@ -496,14 +507,14 @@ def find_comp_in_bom(compname, compver, projver): # The BD API search is inexact. Force our match to be precise. continue if compver == "UNKNOWN": - # We did not have a version specified in the first place - return comp['component'] + # No version specified in SPDX, so treat it as a match + return comp['componentName']+"NOVERSION" # Check component name + version name try: if comp['componentVersionName'].lower() == compver.lower(): - return comp['componentVersion'] + return comp['componentName']+comp['componentVersionName'] except: - # Handle situation where it's missing the version name for some reason + # Handle situation where it's missing the version name print(f"comp {compname} in BOM has no version!") return None return None @@ -641,6 +652,34 @@ def add_to_sbom(proj_version_url, comp_ver_url): logging.error(f"Status code: {response.status_code}") sys.exit(1) +# Get matched component data from component import events +# Input: Summaries URL +# Output: Dictionary containing components added to BOM +# Key= + +# Value= + +def get_matched_comps(summaries_url): + match_dict = {} # dictionary to be returned + + summary_data = bd.get_json(summaries_url) + links = summary_data['_meta']['links'] + for link in links: + # Locate the component-import-events link + if link['rel'] == "component-import-events": + cie_link = link['href'] + break + + # Only consider successful matches + params = { + 'filter': ["eventName:component_mapping_succeeded"] + } + for comp in bd.get_items(cie_link, params=params): + key = comp['importComponentName']+comp['importComponentVersionName'] + val = comp['componentName']+comp['componentVersionName'] + match_dict[key] = val + + return(match_dict) + + def parse_command_args(): parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") @@ -710,7 +749,9 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ upload_sbom_file(spdxfile, projname, vername) # Wait for scan completion. Will exit if it fails. - poll_for_sbom_complete(document.creation_info.name, proj_version_url) + summaries_url = poll_for_sbom_complete(document.creation_info.name, proj_version_url) + # Collect the matched component data for later processing + match_dict = get_matched_comps(summaries_url) # Open unmatched component file to save name, spdxid, version, and # origin/purl for later in json format @@ -812,17 +853,30 @@ def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ else: print(f" No KB match for {package.name} {package.version}") else: - # No external references field was provide + # No external references field was provided nopurl += 1 print(f" No pURL provided for {package.name} {package.version}") - bom_comp = find_comp_in_bom(matchname, matchver, version) + # find_comp_import_events checks the imported name-version + bom_comp = find_comp_import_events(match_dict, package.name, package.version) if bom_comp: + # bom_comp is the matched comp/ver string bom_packages[bom_comp] = bom_packages.get(bom_comp, 0) + 1 packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 bom_matches += 1 - print(f" Found component in BOM: {matchname} {matchver}") + print(f" Found component in bom import-events: {matchname} {matchver}") continue + else: + # Next look for the matchname-matchver in the BOM + # component search. The component name-version may have been + # updated above to reflect the pURL or KB matched name. + bom_comp = find_comp_in_bom(matchname, matchver, version) + if bom_comp: + bom_packages[bom_comp] = bom_packages.get(bom_comp, 0) + 1 + packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 + bom_matches += 1 + print(f" Found component in BOM: {matchname} {matchver}") + continue # If we've gotten this far, the package is not in the BOM. # Now we need to figure out: From 9ca8ba78f670dffc729cb61479870c3d959a8b07 Mon Sep 17 00:00:00 2001 From: Andrew Calder Date: Thu, 23 Nov 2023 16:23:45 +0000 Subject: [PATCH 048/146] use sensible defaults, fix broken requests --- examples/client/generate_sbom.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/examples/client/generate_sbom.py b/examples/client/generate_sbom.py index 70842419..97a6f5e3 100644 --- a/examples/client/generate_sbom.py +++ b/examples/client/generate_sbom.py @@ -45,10 +45,10 @@ class FailedReportDownload(Exception): parser.add_argument("token_file", help="containing access token") parser.add_argument("project_name") parser.add_argument("version_name") -parser.add_argument("-z", "--zip_file_name", default="reports.zip") -parser.add_argument("-t", "--type", type=str, nargs='?', default="SPDX_22", choices=["SPDX_22", "CYCLONEDX_13"], help="Choose the type of SBOM report") +parser.add_argument("-t", "--type", type=str, nargs='?', default="SPDX_23", choices=["SPDX_22", "SPDX_23", "CYCLONEDX_13", "CYCLONEDX_14"], help="Choose the type of SBOM report") parser.add_argument('-r', '--retries', default=4, type=int, help="How many times to retry downloading the report, i.e. wait for the report to be generated") -parser.add_argument('-s', '--sleep_time', default=5, type=int, help="The amount of time to sleep in-between (re-)tries to download the report") +parser.add_argument('-s', '--sleep_seconds', default=60, type=int, help="The amount of time to sleep in-between (re-)tries to download the report") +parser.add_argument('--include-subprojects', dest='include_subprojects', action='store_false', help="whether subprojects should be included") parser.add_argument('--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") args = parser.parse_args() @@ -75,8 +75,8 @@ def download_report(bd_client, location, filename, retries=args.retries): logging.error("Ruh-roh, not sure what happened here") else: logging.debug(f"Failed to retrieve report {report_id}, report status: {report_status}") - logging.debug("Probably not ready yet, waiting 5 seconds then retrying...") - time.sleep(args.sleep_time) + logging.debug(f"Probably not ready yet, waiting {sleep_seconds} seconds then retrying...") + time.sleep(args.sleep_seconds) retries -= 1 download_report(bd_client, location, filename, retries) else: @@ -105,16 +105,19 @@ def download_report(bd_client, location, filename, retries=args.retries): post_data = { 'reportFormat': "JSON", - 'reportType': 'SBOM', - 'sbomType': args.type, + 'sbomType': args.type, + 'includeSubprojects': args.include_subprojects } sbom_reports_url = version['_meta']['href'] + "/sbom-reports" +bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4.json" r = bd.session.post(sbom_reports_url, json=post_data) +if (r.status_code == 403): + logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") r.raise_for_status() location = r.headers.get('Location') assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" logging.debug(f"Created SBOM report of type {args.type} for project {args.project_name}, version {args.version_name} at location {location}") -download_report(bd, location, args.zip_file_name) +download_report(bd, location, f"{args.project_name}({args.version_name}).zip") From 8ebd4ca69c27375804704dd68227bf415140d5df Mon Sep 17 00:00:00 2001 From: Andrew Calder Date: Thu, 23 Nov 2023 16:40:42 +0000 Subject: [PATCH 049/146] fix message --- examples/client/generate_sbom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/generate_sbom.py b/examples/client/generate_sbom.py index 97a6f5e3..08bf3305 100644 --- a/examples/client/generate_sbom.py +++ b/examples/client/generate_sbom.py @@ -75,7 +75,7 @@ def download_report(bd_client, location, filename, retries=args.retries): logging.error("Ruh-roh, not sure what happened here") else: logging.debug(f"Failed to retrieve report {report_id}, report status: {report_status}") - logging.debug(f"Probably not ready yet, waiting {sleep_seconds} seconds then retrying...") + logging.debug(f"Probably not ready yet, waiting {args.sleep_seconds} seconds then retrying...") time.sleep(args.sleep_seconds) retries -= 1 download_report(bd_client, location, filename, retries) From de5507de0b70a4da3b70cf90d6f7eb5fb7ee71ad Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 23 Nov 2023 11:52:19 -0500 Subject: [PATCH 050/146] added example for recursive project deletion --- examples/client/recursive_delete_project.py | 194 ++++++++++++++++++++ examples/client/upload_bdio.py | 1 + 2 files changed, 195 insertions(+) create mode 100644 examples/client/recursive_delete_project.py diff --git a/examples/client/recursive_delete_project.py b/examples/client/recursive_delete_project.py new file mode 100644 index 00000000..107cc0db --- /dev/null +++ b/examples/client/recursive_delete_project.py @@ -0,0 +1,194 @@ +''' +Created: Nov 23, 2023 +Author: mkumykov + +Copyright (c) 2023 - Synopsys, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This script will remove project hierarchies from a Black Duck server. + +usage: python3 recursive_delete_project.py [-h] -u BASE_URL -t TOKEN_FILE [-nv] [-p PROJECT_NAME] [-pv VERSION_NAME] [-pl PROJECT_LIST_FILE] + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -nv, --no-verify Disable TLS certificate verification + -p PROJECT_NAME, --project-name PROJECT_NAME + Project Name + -pv VERSION_NAME, --version-name VERSION_NAME + Version Name + -pl PROJECT_LIST_FILE, --project-list-file PROJECT_LIST_FILE + File containing project name list + +Options -pl and -p can not be used at the same time + +Examples: + +remove project with all sub-projects + + python3 recursive_delete_project.py -u BASE_URL -t TOKEN_FILE -nv -p PROJECT_NAME + +remove project version with all sub-projects, keep sub-projects still in use intact + + python3 recursive_delete_project.py -u BASE_URL -t TOKEN_FILE -nv -p PROJECT_NAME -pv VERSION_NAME + +remove projects listed in a file + + python3 recursive_delete_project.py -u BASE_URL -t TOKEN_FILE -nv -pl PROJECT_LIST_FILE + +''' + +import argparse +import json +import logging +import sys +import arrow + +from blackduck import Client +from pprint import pprint,pformat + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.INFO) +logging.getLogger("urllib3").setLevel(logging.INFO) +logging.getLogger("blackduck").setLevel(logging.INFO) + +strict = False + +def remove_project_structure(project_name): + project = find_project_by_name(project_name) + if not project: + logging.info(f"Project {project_name} does not exist.") + return + versions = bd.get_resource('versions', project) + for version in versions: + remove_project_version_structure(project_name, version['versionName']) + +def remove_project_version_structure(project_name, version_name): + project = find_project_by_name(project_name) + if not project: + logging.info(f"Project {project_name} does not exist.") + return + num_versions = bd.get_resource('versions', project, items=False)['totalCount'] + version = find_project_version_by_name(project,version_name) + if not version: + logging.info(f"Project {project_name} with version {version_name} does not exist.") + return + components = [ + c for c in bd.get_resource('components',version) if c['componentType'] == "SUB_PROJECT" + ] + logging.info(f"Project {project_name}:{version_name} has {len(components)} subprojects") + for component in components: + component_name = component['componentName'] + component_version_name = component['componentVersionName'] + logging.info(f"Removing subproject {component_name} from {project_name}:{version_name}") + component_url = component['_meta']['href'] + response = bd.session.delete(component_url) + logging.info(f"Operation completed with {response}") + remove_project_version_structure(component_name, component_version_name) + logging.info(f"Removing {project_name}:{version_name}") + if num_versions > 1: + response = bd.session.delete(version['_meta']['href']) + else: + response = bd.session.delete(project['_meta']['href']) + logging.info(f"Operation completed with {response}") + +def remove_codelocations_recursively(version): + components = bd.get_resource('components', version) + subprojects = [x for x in components if x['componentType'] == 'SUB_PROJECT'] + logging.info(f"Found {len(subprojects)} subprojects") + unmap_all_codelocations(version) + for subproject in subprojects: + subproject_name = subproject['componentName'] + subproject_version_name = subproject['componentVersionName'] + project = find_project_by_name(subproject_name) + if not project: + logging.info(f"Project {subproject_name} does not exist.") + return + subproject_version = find_project_version_by_name(project, subproject_version_name) + if not subproject_version: + logging.info(f"Project {subproject_name} with version {subversion_name} does not exist.") + return + remove_codelocations_recursively(subproject_version) + +def unmap_all_codelocations(version): + codelocations = bd.get_resource('codelocations',version) + for codelocation in codelocations: + logging.info(f"Unmapping codelocation {codelocation['name']}") + codelocation['mappedProjectVersion'] = "" + response = bd.session.put(codelocation['_meta']['href'], json=codelocation) + pprint (response) + +def find_project_by_name(project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + + + +def parse_command_args(): + + parser = argparse.ArgumentParser("python3 recursive_delete_project.py") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + group = parser.add_mutually_exclusive_group() + group.add_argument("-p", "--project-name", required=False, help="Project Name") + parser.add_argument("-pv", "--version-name", required=False, help="Version Name") + group.add_argument("-pl", "--project-list-file", required=False, help="File containing project name list") + return parser.parse_args() + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + global bd + global scan_params + scan_params = [] + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + logging.info(f"{args}") + + if not (args.project_list_file or args.project_name): + logging.error("Project Name or a File containing Project Names should be specified") + return + + if args.project_list_file: + with open(args.project_list_file) as file: + lines = [line.rstrip() for line in file] + for line in lines: + remove_project_structure(line) + elif args.version_name: + remove_project_version_structure(args.project_name, args.version_name) + else: + remove_project_structure(args.project_name) + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/examples/client/upload_bdio.py b/examples/client/upload_bdio.py index 8df294de..ff505032 100644 --- a/examples/client/upload_bdio.py +++ b/examples/client/upload_bdio.py @@ -65,6 +65,7 @@ def main(): files = {"file": open(args.filename,"rb")} response = bd.session.post("/api/scan/data", files = files) logging.info(response) + logging.info(response.headers) def parse_command_args(): parser = argparse.ArgumentParser(prog = "upload_bdio", description="Uploads BDIO file to a Blackduck server", epilog="Blackduck examples collection") From f01506147a5a62db4ace966cd1a95dcefaccbed8 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 27 Nov 2023 13:17:51 -0500 Subject: [PATCH 051/146] Container image scans now get Project Group assigned .restconfig.json is no longer required for multi-image scans Detect version updated to detect 8 --- .../client/multi-image/manage_project_structure.py | 14 ++++++++++---- .../client/multi-image/scan_docker_image_lite.py | 10 +++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 28467fac..da745cae 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -205,7 +205,7 @@ def create_and_add_child_projects(version, args): repo = next(i, child) tag = next(i,'latest') container_spec = f"{repo}:{tag}" - scan_param = {'image': container_spec, 'project': child, 'version': args.version_name} + scan_param = {'image': container_spec, 'project': child, 'version': args.version_name, 'project_group': args.project_group} if args.clone_from: scan_param['clone_from'] = args.clone_from project = find_project_by_name(child) @@ -267,7 +267,7 @@ def create_project_structure(args): logging.info(f"Checking/Adding subprojects to {args.project_name} : {version['versionName']}") create_and_add_child_projects(version, args) -def scan_container_images(scan_params): +def scan_container_images(scan_params, hub): from scan_docker_image_lite import scan_container_image for params in scan_params: detect_options = (f"--detect.parent.project.name={params['project']} " @@ -276,6 +276,9 @@ def scan_container_images(scan_params): clone_from = params.get('clone_from', None) if clone_from: detect_options += f" --detect.clone.project.version.name={clone_from}" + project_group = params.get('project_group', None) + if project_group: + detect_options += f" --detect.project.group.name={project_group}" scan_container_image( params['image'], None, @@ -283,7 +286,8 @@ def scan_container_images(scan_params): None, params['project'], params['version'], - detect_options + detect_options, + hub=hub ) @@ -320,7 +324,9 @@ def main(): logging.info(f"{pformat(scan_params)}") else: logging.info("Now execution scans") - scan_container_images(scan_params) + from blackduck.HubRestApi import HubInstance + hub = HubInstance(args.base_url, api_token=access_token, insecure=True, debug=False) + scan_container_images(scan_params, hub) if __name__ == "__main__": diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index 442737fb..81fd6a81 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -192,7 +192,8 @@ class Detector(): def __init__(self, hub): # self.detecturl = 'https://blackducksoftware.github.io/hub-detect/hub-detect.sh' # self.detecturl = 'https://detect.synopsys.com/detect.sh' - self.detecturl = 'https://detect.synopsys.com/detect7.sh' + # self.detecturl = 'https://detect.synopsys.com/detect7.sh' + self.detecturl = 'https://detect.synopsys.com/detect8.sh' self.baseurl = hub.config['baseurl'] self.filename = '/tmp/hub-detect.sh' self.token=hub.config['api_token'] @@ -402,9 +403,12 @@ def get_base_layers(self): def scan_container_image( imagespec, grouping=None, base_image=None, dockerfile=None, - project_name=None, project_version=None, detect_options=None): + project_name=None, project_version=None, detect_options=None, hub=None): - hub = HubInstance() + if hub: + hub = hub + else: + hub = HubInstance() scanner = ContainerImageScanner( hub, imagespec, grouping=grouping, base_image=base_image, dockerfile=dockerfile, detect_options=detect_options) From d91c7fec056f6f34cd1fd768a3e3b878747adf6d Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 1 Dec 2023 16:13:49 -0500 Subject: [PATCH 052/146] unused code removed --- examples/client/recursive_delete_project.py | 26 --------------------- 1 file changed, 26 deletions(-) diff --git a/examples/client/recursive_delete_project.py b/examples/client/recursive_delete_project.py index 107cc0db..921d9cf9 100644 --- a/examples/client/recursive_delete_project.py +++ b/examples/client/recursive_delete_project.py @@ -106,32 +106,6 @@ def remove_project_version_structure(project_name, version_name): response = bd.session.delete(project['_meta']['href']) logging.info(f"Operation completed with {response}") -def remove_codelocations_recursively(version): - components = bd.get_resource('components', version) - subprojects = [x for x in components if x['componentType'] == 'SUB_PROJECT'] - logging.info(f"Found {len(subprojects)} subprojects") - unmap_all_codelocations(version) - for subproject in subprojects: - subproject_name = subproject['componentName'] - subproject_version_name = subproject['componentVersionName'] - project = find_project_by_name(subproject_name) - if not project: - logging.info(f"Project {subproject_name} does not exist.") - return - subproject_version = find_project_version_by_name(project, subproject_version_name) - if not subproject_version: - logging.info(f"Project {subproject_name} with version {subversion_name} does not exist.") - return - remove_codelocations_recursively(subproject_version) - -def unmap_all_codelocations(version): - codelocations = bd.get_resource('codelocations',version) - for codelocation in codelocations: - logging.info(f"Unmapping codelocation {codelocation['name']}") - codelocation['mappedProjectVersion'] = "" - response = bd.session.put(codelocation['_meta']['href'], json=codelocation) - pprint (response) - def find_project_by_name(project_name): params = { 'q': [f"name:{project_name}"] From 95d90d1a9d3204de2bc64887f35235251f05f98f Mon Sep 17 00:00:00 2001 From: Andrew Calder Date: Thu, 7 Dec 2023 10:14:11 +0000 Subject: [PATCH 053/146] content-type fix --- examples/client/generate_sbom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/generate_sbom.py b/examples/client/generate_sbom.py index 08bf3305..4a0fef00 100644 --- a/examples/client/generate_sbom.py +++ b/examples/client/generate_sbom.py @@ -110,7 +110,7 @@ def download_report(bd_client, location, filename, retries=args.retries): } sbom_reports_url = version['_meta']['href'] + "/sbom-reports" -bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4.json" +bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json" r = bd.session.post(sbom_reports_url, json=post_data) if (r.status_code == 403): logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") From f3b7b881db2a2395d6f13bc9995986e9090fb6c8 Mon Sep 17 00:00:00 2001 From: dnichol Date: Thu, 4 Jan 2024 16:43:55 +0000 Subject: [PATCH 054/146] Example creating an API token --- examples/create_api_token.py | 67 ++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 examples/create_api_token.py diff --git a/examples/create_api_token.py b/examples/create_api_token.py new file mode 100644 index 00000000..60db1c81 --- /dev/null +++ b/examples/create_api_token.py @@ -0,0 +1,67 @@ +''' +Created on January 1, 2024 + +@author: dnichol + +Create a new user + +To use this script. Firstly create a .restconfig.json file with either API token or basic auth (username/password) as per the examples : +https://github.com/blackducksoftware/hub-rest-api-python/blob/master/restconfig.json.example +https://github.com/blackducksoftware/hub-rest-api-python/blob/master/restconfig.json.api_token.example + +Then to run: +python create_api_token.py MyToken "My Token Description" + +It will output the token that is generated. If you would like the token to be read only add the -r flag to the command line. + +''' +import argparse +import json +import logging +from pprint import pprint +import sys + +from blackduck.HubRestApi import HubInstance + + +parser = argparse.ArgumentParser("Create an API token") +parser.add_argument("name") +parser.add_argument("description") +parser.add_argument("-r", "--readonly", action='store_true') + + +args = parser.parse_args() + +logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) + +hub = HubInstance() + +scope = ["read"] + +if not args.readonly : + scope = ["read", "write"] + +post_data = { + "name" : args.name, + "description" : args.description, + "scopes" : scope +} + +current_user = hub.get_current_user() +add_token_url = hub.get_link(current_user, "api-tokens") + +response = hub.execute_post(add_token_url, data=post_data) +if response.status_code == 201: + token_obj = response.json() + token=token_obj['token'] + logging.info("Added API token {} = {}".format(args.name, token)) +else: + logging.error("Failed to add API token {}, status code was {}".format( + args.name, response.status_code)) + + + + + From 03ec6d2f660c126e8be777a673e9bb14ae85cfeb Mon Sep 17 00:00:00 2001 From: dnichol Date: Thu, 4 Jan 2024 16:51:23 +0000 Subject: [PATCH 055/146] Example creating an API token --- examples/create_api_token.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/create_api_token.py b/examples/create_api_token.py index 60db1c81..a0315ed7 100644 --- a/examples/create_api_token.py +++ b/examples/create_api_token.py @@ -3,7 +3,7 @@ @author: dnichol -Create a new user +Create an API token with either readonly or read and write access. To use this script. Firstly create a .restconfig.json file with either API token or basic auth (username/password) as per the examples : https://github.com/blackducksoftware/hub-rest-api-python/blob/master/restconfig.json.example From defe2f2580a3789a40084fb9994e19e0bf93899f Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 12 Jan 2024 11:05:16 -0500 Subject: [PATCH 056/146] added quotation to handle space in GroupName --- examples/client/multi-image/manage_project_structure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index da745cae..96af1121 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -278,7 +278,7 @@ def scan_container_images(scan_params, hub): detect_options += f" --detect.clone.project.version.name={clone_from}" project_group = params.get('project_group', None) if project_group: - detect_options += f" --detect.project.group.name={project_group}" + detect_options += f" --detect.project.group.name=\"{project_group}\"" scan_container_image( params['image'], None, From b17a1978c7da791096cb9d9a5eeaaed9f1f6fa2b Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 18 Jan 2024 14:25:08 -0500 Subject: [PATCH 057/146] space in group name handling fixed --- examples/client/multi-image/generate-clone.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/multi-image/generate-clone.sh b/examples/client/multi-image/generate-clone.sh index 7337da2e..6b3cbac6 100644 --- a/examples/client/multi-image/generate-clone.sh +++ b/examples/client/multi-image/generate-clone.sh @@ -11,5 +11,5 @@ nlv:testcontainer:2.4" COMMAND="python3 examples/client/multi-image/manage_project_structure.py" -$COMMAND -u $BD_URL -t token -nv -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ +$COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ From 4582c73cd8e9fc5e4066fd5da622be5a8e0b90b2 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 18 Jan 2024 14:53:13 -0500 Subject: [PATCH 058/146] unmap all codelocations from a project version example --- examples/client/unmap_codelocations.py | 66 ++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 examples/client/unmap_codelocations.py diff --git a/examples/client/unmap_codelocations.py b/examples/client/unmap_codelocations.py new file mode 100644 index 00000000..a3d080bd --- /dev/null +++ b/examples/client/unmap_codelocations.py @@ -0,0 +1,66 @@ +''' +Created on Jan 18, 2024 + +@author: kumykov + +Unmap codelocations from a project version + +''' + +from blackduck import Client + +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument("-u", "--bd-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", help="File name of a file containing access token") +parser.add_argument("-nv", '--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("project_name") +parser.add_argument("version_name") + +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) + +params = { + 'q': [f"name:{args.project_name}"] +} +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +params = { + 'q': [f"versionName:{args.version_name}"] +} +versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name] +assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}" +version = versions[0] + +logging.debug(f"Found {project['name']}:{version['versionName']}") + +codelocations = bd.get_resource('codelocations', version) + +for codelocation in codelocations: + logging.debug(f"Un-mapping code location {codelocation['name']}") + url = codelocation['_meta']['href'] + codelocation['mappedProjectVersion'] = None + result = bd.session.put(url, json=codelocation) + logging.info(f"Code location '{codelocation['name']}' unmap status {result}") \ No newline at end of file From 39e6d344edf7026e9bff1cae1d6a5ff34884c447 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 25 Jan 2024 14:56:03 -0500 Subject: [PATCH 059/146] . --- examples/client/update_project_settings.py | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 examples/client/update_project_settings.py diff --git a/examples/client/update_project_settings.py b/examples/client/update_project_settings.py new file mode 100644 index 00000000..790158be --- /dev/null +++ b/examples/client/update_project_settings.py @@ -0,0 +1,65 @@ +''' +Created on Jan 18, 2024 + +@author: kumykov + +Update project settings script. + +This script will modify project settings accessible via API + +Parameter list and their default values: + "customSignatureEnabled" : false, + "customSignatureDepth" : "5", + "unmatchedFileRetentionEnabled" : false, + +''' + +from blackduck import Client + +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument("-u", "--bd-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", help="File name of a file containing access token") +parser.add_argument("-nv", '--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("-cse", '--custom-signature-enabled', dest='cs_enable', action='store_true', help="enable custom signature flag") +parser.add_argument("-csd", '--custom-signature-depth', dest='cs_depth', required=False, default="5", help="set custom signature depth") +parser.add_argument("-ruf", '--retain-unmatched-files', dest='retain_uf', action='store_true', help="set retain unmatched files flag") +parser.add_argument("project_name") + +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) +pprint (args.project_name) +params = { + 'q': [f"name:{args.project_name}"] +} +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +url = project['_meta']['href'] +project['customSignatureEnabled'] = args.cs_enable +project['customSignatureDepth'] = args.cs_depth +project['unmatchedFileRetentionEnabled'] = args.retain_uf + +response = bd.session.put(url, json=project) +logging.info(f"Project setting update status {response}") \ No newline at end of file From aa2067717969389e6e6a034d95c4d8b20473cb71 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 25 Jan 2024 14:58:51 -0500 Subject: [PATCH 060/146] . --- examples/client/update_project_settings.py | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 examples/client/update_project_settings.py diff --git a/examples/client/update_project_settings.py b/examples/client/update_project_settings.py new file mode 100644 index 00000000..b93a18e1 --- /dev/null +++ b/examples/client/update_project_settings.py @@ -0,0 +1,65 @@ +''' +Created on Jan 18, 2024 + +@author: kumykov + +Update project settings script. + +This script will modify project settings accessible via API + +Parameter list and their default values: + "customSignatureEnabled" : false, + "customSignatureDepth" : "5", + "unmatchedFileRetentionEnabled" : false, + +''' + +from blackduck import Client + +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument("-u", "--bd-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", help="File name of a file containing access token") +parser.add_argument("-nv", '--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("-cse", '--custom-signature-enabled', dest='cs_enable', action='store_true', help="enable custom signature flag") +parser.add_argument("-csd", '--custom-signature-depth', dest='cs_depth', required=False, default="5", help="set custom signature depth") +parser.add_argument("-ruf", '--retain-unmatched-files', dest='retain_uf', action='store_true', help="set retain unmatched files flag") +parser.add_argument("project_name") + +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) +pprint (args.project_name) +params = { + 'q': [f"name:{args.project_name}"] +} +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +url = project['_meta']['href'] +project['customSignatureEnabled'] = args.cs_enable +project['customSignatureDepth'] = args.cs_depth +project['unmatchedFileRetentionEnabled'] = args.retain_uf + +response = bd.session.put(url, json=project) +logging.info(f"Project setting update status {response}") From cea338f140514609513ea51c7ad838572a2bbccc Mon Sep 17 00:00:00 2001 From: Dinesh Date: Tue, 30 Jan 2024 12:06:21 +0100 Subject: [PATCH 061/146] add script for getting total scan size --- .../get_project_version_total_scan_size.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 examples/get_project_version_total_scan_size.py diff --git a/examples/get_project_version_total_scan_size.py b/examples/get_project_version_total_scan_size.py new file mode 100644 index 00000000..ce9edb9d --- /dev/null +++ b/examples/get_project_version_total_scan_size.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from blackduck.HubRestApi import HubInstance + +import argparse +import json + +parser = argparse.ArgumentParser() +parser.add_argument("project_name") +parser.add_argument("version_name") +args = parser.parse_args() + +hub = HubInstance() + +project = hub.get_project_by_name(args.project_name) +if project: + version = hub.get_version_by_name(project, args.version_name) + + if version: + codelocation_url = hub.get_link(version, "codelocations") + response = hub.execute_get(codelocation_url) + if response.status_code == 200: + # codelocation and scan are synonymous + codelocation_info = response.json().get('items', []) + if codelocation_info: + most_recent_scan = max([cl['updatedAt'] for cl in codelocation_info]) + oldest_scan = min([cl['createdAt'] for cl in codelocation_info]) + number_scans = len(codelocation_info) + else: + number_scans = 0 + oldest_scan = most_recent_scan = "Not applicable" + + resultString = json.dumps( + { + 'scans': codelocation_info, + 'number_scans': number_scans, + 'most_recent_scan': most_recent_scan, + 'oldest_scan': oldest_scan + }) + resultArray = json.loads(resultString) + print(f"Number of scan performed {resultArray.get('number_scans')}") + scansArray = resultArray.get('scans') + sizeTrack=0 + for i, scan in enumerate(scansArray): + print(f"Scan {i+1} name:{scan['name']}") + sizeinbytes= scan['scanSize'] + size=sizeinbytes/1024/1024 + sizeTrack=sizeTrack+sizeinbytes + print(f" size:{'{0:.2f}'.format(size)} MB") + + totalsize=sizeTrack/1024/1024 + print(f"Project Total size:{'{0:.2f}'.format(totalsize)} MB") + + + # print(json.dumps( + # { + # 'scans': codelocation_info, + # 'number_scans': number_scans, + # 'most_recent_scan': most_recent_scan, + # 'oldest_scan': oldest_scan + # })) + else: + print("Failed to retrieve the codelocation (aka scan) info, response code was {}".format(response.status_code)) + else: + print("Could not find the version {} in project {}".format(args.version_name, args.project_name)) +else: + print("Could not find the project {}".format(args.project_name)) \ No newline at end of file From 3810a885e3d449f00fe96d27035f3f7518331acc Mon Sep 17 00:00:00 2001 From: Dinesh Date: Tue, 30 Jan 2024 14:41:52 +0100 Subject: [PATCH 062/146] added limit --- examples/get_project_version_total_scan_size.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/get_project_version_total_scan_size.py b/examples/get_project_version_total_scan_size.py index ce9edb9d..d6ea0170 100644 --- a/examples/get_project_version_total_scan_size.py +++ b/examples/get_project_version_total_scan_size.py @@ -18,6 +18,7 @@ if version: codelocation_url = hub.get_link(version, "codelocations") + codelocation_url += "?limit={}".format(10000) response = hub.execute_get(codelocation_url) if response.status_code == 200: # codelocation and scan are synonymous From b5f1b6318caa44a41eba414ed8e7f7a0addca04a Mon Sep 17 00:00:00 2001 From: Dinesh Date: Tue, 30 Jan 2024 14:45:17 +0100 Subject: [PATCH 063/146] added print --- examples/get_project_version_total_scan_size.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/get_project_version_total_scan_size.py b/examples/get_project_version_total_scan_size.py index d6ea0170..2dd192cb 100644 --- a/examples/get_project_version_total_scan_size.py +++ b/examples/get_project_version_total_scan_size.py @@ -50,8 +50,9 @@ print(f" size:{'{0:.2f}'.format(size)} MB") totalsize=sizeTrack/1024/1024 + print("===============================") print(f"Project Total size:{'{0:.2f}'.format(totalsize)} MB") - + print("===============================") # print(json.dumps( # { From 6be9333bdef34b8024ab340b9a2d32705399654a Mon Sep 17 00:00:00 2001 From: Dinesh Date: Tue, 30 Jan 2024 14:54:20 +0100 Subject: [PATCH 064/146] increased limit --- examples/get_project_version_total_scan_size.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/get_project_version_total_scan_size.py b/examples/get_project_version_total_scan_size.py index 2dd192cb..e29ab86f 100644 --- a/examples/get_project_version_total_scan_size.py +++ b/examples/get_project_version_total_scan_size.py @@ -18,7 +18,7 @@ if version: codelocation_url = hub.get_link(version, "codelocations") - codelocation_url += "?limit={}".format(10000) + codelocation_url += "?limit={}".format(1000000) response = hub.execute_get(codelocation_url) if response.status_code == 200: # codelocation and scan are synonymous From 6e96217668b8f2f408363ba6edfcca33ded277fe Mon Sep 17 00:00:00 2001 From: varunkpedapati <76003520+varunkpedapati@users.noreply.github.com> Date: Wed, 31 Jan 2024 09:58:55 -0800 Subject: [PATCH 065/146] Add files via upload --- examples/client/update_component_version.py | 205 ++++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 examples/client/update_component_version.py diff --git a/examples/client/update_component_version.py b/examples/client/update_component_version.py new file mode 100644 index 00000000..df56cf46 --- /dev/null +++ b/examples/client/update_component_version.py @@ -0,0 +1,205 @@ +''' +Created on Jan 22, 2024 + +@author: pedapati + +Update component version info for BOM Components with Unknown Versions based on matched filename in a given project version + +''' + +from blackduck import Client + +import requests +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +import urllib3 +import urllib.parse + +NAME = 'update_component_version.py' +VERSION = '2024-01-22' + +print(f'{NAME} ({VERSION}). Copyright (c) 2023 Synopsys, Inc.') + + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument("-u", "--bd-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", help="File name of a file containing access token") +parser.add_argument("-nv", '--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("project_name") +parser.add_argument("version_name") + +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) + +params = { + 'q': [f"name:{args.project_name}"] +} +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] +project_id = project["_meta"]["href"].split("/")[-1] +print("Project ID: " + project_id) + +params = { + 'q': [f"versionName:{args.version_name}"] +} +versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name] +assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}" +version = versions[0] +version_id = version["_meta"]["href"].split("/")[-1] +print("Version ID: " + version_id) + +logging.debug(f"Found {project['name']}:{version['versionName']}") + +def update_bom_unknown_versions(bd, project_id, version_id): + limit = 1000 + offset = 0 + paginated_url = f"{bd.base_url}/api/projects/{project_id}/versions/{version_id}/components?limit={limit}&offset={offset}&filter=unknownVersion:true" + print("Looking for BOM Components with Unknown Versions: " + paginated_url) + print() + components_json = bd.session.get(paginated_url).json() + total = str(components_json["totalCount"]) + print("Found " + total + " components with unknown versions") + print() + for component in components_json["items"]: + comp_name =component["componentName"] + print("Processing Component: " + comp_name) + comp_url = component["component"] + comp_bom_url = component["_meta"]["href"] + matched_files_url = component["_meta"]["href"] + "/matched-files" + matched_file_json = bd.session.get(matched_files_url).json() + archivecontext = matched_file_json["items"][0]["filePath"]["archiveContext"] + filename = matched_file_json["items"][0]["filePath"]["fileName"] + ## Extract Component Name and Version from archivecontext to do a KB lookup + archive_strip = archivecontext.strip("/,!") + archive_partition = archive_strip.rpartition("-") + archive_final_list = archive_partition[0].rpartition("-") + kb_file_lookup_name = archive_final_list[0] + kb_comp_lookup_version = archive_final_list[2] + print("Processing Component Version: " + kb_comp_lookup_version) + ## KB Lookup via Component Name + components_url = bd.base_url + "/api/components/autocomplete" + query = { "q": comp_name, + "filter": "componentType:kb_component" + } + url = f"{components_url}?{urllib.parse.urlencode(query)}" + headers = {'Accept': '*/*'} + name_match = bd.session.get(url, headers=headers).json() + # Filtering results for exact name match + exact_name_match = [x for x in name_match['items'] if x['name']==comp_name] + if len(exact_name_match) == 0 : + logging.debug(f"Component {comp_name} is not found in the KB") + return + else: + logging.debug(f"Component {comp_name} is found in the KB") + if kb_comp_lookup_version: + first_match_successful = False + # second_match_successful = False + for match in exact_name_match: # handling OSS components that share same name + url = match['_meta']['href']+'/versions?q=versionName:' + kb_comp_lookup_version + headers = {'Accept': 'application/vnd.blackducksoftware.summary-1+json'} + # Producing version matches + version_match = bd.session.get(url, headers=headers).json() + if version_match['totalCount'] > 0: + print(version_match["items"][0]["versionName"]) + print("Found version: " + kb_comp_lookup_version + " in the KB for component " + comp_name) + print("Updating component version for " + comp_name + " to " + kb_comp_lookup_version ) + # component_url = version_match[] + # print(version_match) + component_version_url = version_match['items'][0]['_meta']['href'] + component_url = component_version_url[:component_version_url.index("versions")-1] + # print(component_url) + post_data = {"component": component_url, "componentVersion": component_version_url} + headers = {'Accept': 'application/vnd.blackducksoftware.bill-of-materials-6+json', 'Content-Type': 'application/vnd.blackducksoftware.bill-of-materials-6+json'} + response = bd.session.put(comp_bom_url, headers=headers, data=json.dumps(post_data)) + # print(response) + if response.status_code == 200: + message = f"{response}" + print("Successfully updated " + comp_name + " with version " + kb_comp_lookup_version) + else: + message = f"{response.json()}" + logging.debug(f"Updating BOM component {comp_name} {kb_comp_lookup_version} failed with: {message}") + first_match_successful = True + print("### Proceeding to next component") + print() + break + else: + print("No matching version " + kb_comp_lookup_version + " found for " + comp_name) + if not first_match_successful: + ## Trying to locate component name using source archive name + print("Proceeding to KB lookup via matched file name: " + kb_file_lookup_name) + components_url = bd.base_url + "/api/components/autocomplete" + query = { "q": kb_file_lookup_name, + "filter": "componentType:kb_component" + } + url = f"{components_url}?{urllib.parse.urlencode(query)}" + headers = {'Accept': '*/*'} + name_match = bd.session.get(url, headers=headers).json() + # Filtering results for exact name match + exact_name_match = [x for x in name_match['items'] if x['name']==kb_file_lookup_name] + if len(exact_name_match) == 0 : + logging.debug(f"File Match KB Component {kb_file_lookup_name} is not found in the KB") + print("### Proceeding to next component") + print() + continue + else: + logging.debug(f"File Match KB Component {kb_file_lookup_name} is found in the KB") + if kb_comp_lookup_version: + for match in exact_name_match: # handling OSS components that share same name + url = match['_meta']['href']+'/versions?q=versionName:' + kb_comp_lookup_version + headers = {'Accept': 'application/vnd.blackducksoftware.summary-1+json'} + # Producing version matches + version_match = bd.session.get(url, headers=headers).json() + if version_match['totalCount'] > 0: + print(version_match["items"][0]["versionName"]) + print("Found version: " + kb_comp_lookup_version + " in the KB for component " + kb_file_lookup_name) + print("Updating component version for " + kb_file_lookup_name + " to " + kb_comp_lookup_version ) + # component_url = version_match[] + # print(version_match) + component_version_url = version_match['items'][0]['_meta']['href'] + component_url = component_version_url[:component_version_url.index("versions")-1] + # print(component_url) + post_data = {"component": component_url, "componentVersion": component_version_url} + headers = {'Accept': 'application/vnd.blackducksoftware.bill-of-materials-6+json', 'Content-Type': 'application/vnd.blackducksoftware.bill-of-materials-6+json'} + response = bd.session.put(comp_bom_url, headers=headers, data=json.dumps(post_data)) + # print(response) + if response.status_code == 200: + message = f"{response}" + print("Successfully updated " + kb_file_lookup_name + " with version " + kb_comp_lookup_version) + # second_match_successful = True + else: + message = f"{response.json()}" + logging.debug(f"Updating BOM component {kb_file_lookup_name} {kb_comp_lookup_version} failed with: {message}") + print("### Proceeding to next component") + print() + break + else: + print("No matching version " + kb_comp_lookup_version + " found for " + kb_file_lookup_name) + print("### Proceeding to next component") + print() + + + +bom = update_bom_unknown_versions(bd, project_id, version_id) + + + From a9a271bf6f2e8de47a530159a32529eac050274a Mon Sep 17 00:00:00 2001 From: varunkpedapati <76003520+varunkpedapati@users.noreply.github.com> Date: Wed, 31 Jan 2024 10:00:04 -0800 Subject: [PATCH 066/146] Add files via upload --- .../copy_kb_component_status_updates.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 examples/client/copy_kb_component_status_updates.py diff --git a/examples/client/copy_kb_component_status_updates.py b/examples/client/copy_kb_component_status_updates.py new file mode 100644 index 00000000..7bf5a763 --- /dev/null +++ b/examples/client/copy_kb_component_status_updates.py @@ -0,0 +1,110 @@ +''' +Created on Jan 29, 2024 +@author: pedapati + +Copyright (C) 2024 Synopsys, Inc. +https://www.synopsys.com + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +This program will Export KB & KB Modified Component Status updates from one Black Duck Server and Import onto another Black Duck Server +''' + +from blackduck import Client + +import requests +import argparse +import json +import logging +import sys +import time +from pprint import pprint + + +NAME = 'copy_kb_component_status_updates.py' +VERSION = '2024-01-29' + +print(f'{NAME} ({VERSION}). Copyright (c) 2023 Synopsys, Inc.') + + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + + +def parse_command_args(): + + parser = argparse.ArgumentParser(sys.argv[0]) + parser.add_argument("-su", "--source-bd-url", required=True, help="Source BD server URL to copy KB component adjustments from e.g. https://your.blackduck.url") + parser.add_argument("-st", "--source-token-file", required=True, help="File containing Source BD access token") + parser.add_argument("-du", "--dest-bd-url", required=True, help="Destination BD server URL to apply KB component adjustments to e.g. https://your.blackduck.url") + parser.add_argument("-dt", "--dest-token-file", required=True, help="File containing Destination BD access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + return parser.parse_args() + + +def main(): + args = parse_command_args() + ## Step 1 Source Black Duck Server - Authentication + with open(args.source_token_file, 'r') as tf: + access_token = tf.readline().strip() + bd1 = Client(base_url=args.source_bd_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + ## Step 2 Destination Black Duck Server - Authentication + with open(args.dest_token_file, 'r') as tf: + access_token1 = tf.readline().strip() + bd2 = Client(base_url=args.dest_bd_url, token=access_token1, verify=args.no_verify, timeout=60.0, retries=4) + ## Step 3 Source Black Duck Server - Get KB Components with Status Updates + headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} + get_comp_url = f"{bd1.base_url}/api/components?filter=componentApprovalStatus%3Ain_review&filter=componentApprovalStatus%3Areviewed&filter=componentApprovalStatus%3Aapproved&filter=componentApprovalStatus%3Alimited_approval&filter=componentApprovalStatus%3Arejected&filter=componentApprovalStatus%3Adeprecated&filter=componentSource%3Akb_and_kb_modified&limit=25&offset=0" + get_comp_json = bd1.session.get(get_comp_url, headers=headers).json() + total = str(get_comp_json["totalCount"]) + print("Found " + total + " KB components with status updates") + print() + for component in get_comp_json["items"]: + comp_name = component['name'] + # comp_url = component['url'] + comp_status = component['approvalStatus'] + comp_url = component['_meta']['href'] + comp_id = comp_url.split("/")[-1] + print("Updating KB Component " + comp_name + " status to " + comp_status) + ## Step 4 Destination Black Duck Server - Update KB Components with Status Updates + headers = {'Content-Type': 'application/vnd.blackducksoftware.component-detail-4+json', 'Accept': 'application/vnd.blackducksoftware.component-detail-4+json'} + put_data = {"name": comp_name, "approvalStatus": comp_status} + put_comp_url = f"{bd2.base_url}/api/components/{comp_id}" + update_comp_results=bd2.session.put(put_comp_url, headers=headers, data=json.dumps(put_data)) + if update_comp_results.status_code == 200: + message = f"{update_comp_results}" + print("Successfully updated status of " + comp_name + " to " + comp_status) + print() + else: + message = f"{update_comp_results.json()}" + print("Updating status FAILED with error message:") + print() + logging.debug({message}) + print() + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file From f4c620064b84c8f8ffd389b5ee3cdea2c756554a Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Sun, 4 Feb 2024 21:47:11 -0500 Subject: [PATCH 067/146] Sub-project specification with excel file --- .../multi-image/manage_project_structure.py | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 96af1121..80f546f3 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -50,6 +50,8 @@ Project Version Name -sp SUBPROJECT_LIST, --subproject-list SUBPROJECT_LIST List of subprojects to generate with subproject:container:tag + -ssf SUBPROJECT_SPEC_FILE, --subproject-spec-file SUBPROJECT_SPEC_FILE + Excel file containing subproject specification -nv, --no-verify Disable TLS certificate verification -rm, --remove Remove project structure with all subprojects (DANGEROUS!) --clone-from CLONE_FROM @@ -60,6 +62,11 @@ if container name omited it will be set to subproject if tag omited it would be set to 'latest' +Subprojects an be specified in excel file with -ssf --subproject-spec-file parameter. +Excel file should contain one worksheet with first row containing column names as following: +Container Name, Image ID, Version, Project Name +and subsequent rows containing data + Container image name scanned will be written into project version nickname field @@ -197,9 +204,36 @@ def add_component_to_version_bom(child_version, version): data = { 'component': child_version['_meta']['href']} return bd.session.post(url, json=data) +def get_child_spec_list(args): + if args.subproject_list: + return args.subproject_list.split(',') + else: + print("processing excel") + import openpyxl + wb = openpyxl.load_workbook(args.subproject_spec_file) + ws = wb.active + project_list = [] + row_number = 0 + for row in ws.values: + row_number += 1 + if (row_number == 1 and + row[0] == 'Container Name' and + row[1] == 'Image ID' and + row[2] == 'Version' and + row[3] == 'Project Name'): + print("File Format checks out (kind of)") + continue + elif row_number > 1: + project_list.append(f"{row[3]}:{row[0]}:{row[2]}") + else: + logging.error(f"Could not parse input file {args.subproject_spec_file}") + sys.exit(1) + return (project_list) + def create_and_add_child_projects(version, args): version_url = version['_meta']['href'] + '/components' - for child_spec in [x.split(':') for x in args.subproject_list.split(",")]: + child_spec_list = get_child_spec_list(args) + for child_spec in [x.split(':') for x in child_spec_list]: i = iter(child_spec) child = next(i) repo = next(i, child) @@ -299,7 +333,9 @@ def parse_command_args(): parser.add_argument("-pg", "--project_group", required=False, default='Multi-Image', help="Project Group to be used") parser.add_argument("-p", "--project-name", required=True, help="Project Name") parser.add_argument("-pv", "--version-name", required=True, help="Project Version Name") - parser.add_argument("-sp", "--subproject-list", required=False, help="List of subprojects to generate with subproject:container:tag") + group = parser.add_mutually_exclusive_group() + group.add_argument("-sp", "--subproject-list", required=False, help="List of subprojects to generate with subproject:container:tag") + group.add_argument("-ssf", "--subproject-spec-file", required=False, help="Excel file containing subproject specification") parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") parser.add_argument("-rm", "--remove", action='store_true', required=False, help="Remove project structure with all subprojects (DANGEROUS!)") parser.add_argument("--clone-from", required=False, help="Main project version to use as template for cloning") From 107d1df06db28ff6c13734a0ecd3da001e07419d Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Sun, 4 Feb 2024 21:48:54 -0500 Subject: [PATCH 068/146] Sub-project specification with excel file --- examples/client/multi-image/manage_project_structure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 80f546f3..36810b8e 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -208,7 +208,7 @@ def get_child_spec_list(args): if args.subproject_list: return args.subproject_list.split(',') else: - print("processing excel") + logging.info(f"Processing excel file {args.subproject_spec_file}") import openpyxl wb = openpyxl.load_workbook(args.subproject_spec_file) ws = wb.active @@ -221,7 +221,7 @@ def get_child_spec_list(args): row[1] == 'Image ID' and row[2] == 'Version' and row[3] == 'Project Name'): - print("File Format checks out (kind of)") + logging.info(f"File Format checks out (ind of)") continue elif row_number > 1: project_list.append(f"{row[3]}:{row[0]}:{row[2]}") From 9ce3fb08fb08680db13234bfa0eb403fac6c200a Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Sun, 4 Feb 2024 21:50:52 -0500 Subject: [PATCH 069/146] Sub-project specification with excel file --- examples/client/multi-image/manage_project_structure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 36810b8e..8d7c5472 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -221,7 +221,7 @@ def get_child_spec_list(args): row[1] == 'Image ID' and row[2] == 'Version' and row[3] == 'Project Name'): - logging.info(f"File Format checks out (ind of)") + logging.info(f"File Format checks out (kind of)") continue elif row_number > 1: project_list.append(f"{row[3]}:{row[0]}:{row[2]}") From 6b50f9e90c89e210f8c084e43833ddb316dc5733 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 9 Feb 2024 10:22:33 -0500 Subject: [PATCH 070/146] excel input test --- examples/client/multi-image/generate-clone.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/examples/client/multi-image/generate-clone.sh b/examples/client/multi-image/generate-clone.sh index 6b3cbac6..e8a0c36f 100644 --- a/examples/client/multi-image/generate-clone.sh +++ b/examples/client/multi-image/generate-clone.sh @@ -9,7 +9,12 @@ le:testcontainer:2.4,\ login-app-ui:testcontainer:2.4,\ nlv:testcontainer:2.4" +SPECFILE=~/Documents/Ciena/excelparameters/BP_SampleProduct.xlsx + +ls -l $SPECFILE + COMMAND="python3 examples/client/multi-image/manage_project_structure.py" -$COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ +# $COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ +$COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@ From dc9b09090248e8844eef456a48669b316e55dfe8 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Sun, 11 Feb 2024 19:49:41 -0500 Subject: [PATCH 071/146] fixed existing subprojects not added --- examples/client/multi-image/manage_project_structure.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 8d7c5472..3a734e18 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -252,6 +252,14 @@ def create_and_add_child_projects(version, args): else: logging.info(f"Child project {project['name']} with version {args.version_name} found.") logging.info(f"Recursively removing codelocations for {project['name']} with version {args.version_name} ") + try: + logging.info(f"Adding project {child} {args.version_name} to the parent project") + child_version_url = version['_meta']['href'] + response = bd.session.post(version_url,json={'component': child_version_url}) + logging.info(f"Adding {child} : {args.version_name} to parent project completed with {response}") + except Exception as e: + logging.info(f"Adding {child} : {args.version_name} to parent project completed with exception {e}") + remove_codelocations_recursively(version) else: response = create_project_version(child,args.version_name, args, nickname=container_spec) From 7f1768879c504f69c4a251bdbe77d9a3ee96b2ad Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 29 Feb 2024 12:44:31 -0500 Subject: [PATCH 072/146] Fixed emty layer count discrepancy --- examples/client/generate_sbom.py | 2 +- examples/client/multi-image/generate-clone.sh | 4 +-- .../multi-image/scan_docker_image_lite.py | 25 +++++++++++++++++-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/examples/client/generate_sbom.py b/examples/client/generate_sbom.py index 08bf3305..4a0fef00 100644 --- a/examples/client/generate_sbom.py +++ b/examples/client/generate_sbom.py @@ -110,7 +110,7 @@ def download_report(bd_client, location, filename, retries=args.retries): } sbom_reports_url = version['_meta']['href'] + "/sbom-reports" -bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4.json" +bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json" r = bd.session.post(sbom_reports_url, json=post_data) if (r.status_code == 403): logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") diff --git a/examples/client/multi-image/generate-clone.sh b/examples/client/multi-image/generate-clone.sh index e8a0c36f..ca90b853 100644 --- a/examples/client/multi-image/generate-clone.sh +++ b/examples/client/multi-image/generate-clone.sh @@ -15,6 +15,6 @@ ls -l $SPECFILE COMMAND="python3 examples/client/multi-image/manage_project_structure.py" -# $COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ -$COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@ +$COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ +# $COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@ diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index 81fd6a81..84adcff5 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -240,8 +240,29 @@ def __init__( if detect_options: self.extra_options = detect_options.split(" ") print ("<--{}-->".format(self.grouping)) - - def prepare_container_image(self): + + def prepare_container_image_old(self): + self.docker.initdir() + self.docker.pull_container_image(self.container_image_name) + self.docker.save_container_image(self.container_image_name) + self.docker.unravel_container() + # result = self.docker.get_container_image_history(self.container_image_name) + history = self.docker.read_config()['history'] + layer_count = 0 + history_grouping = '' + for item in history: + if not item.get('empty_layer', None): + layer_count += 1 + match = re.search('echo (.+?)_group_end', item['created_by']) + if match: + found = match.group(1) + if len(history_grouping): + history_grouping += ',' + history_grouping += str(layer_count) + ":" + found + if len(history_grouping) and self.grouping == '1024:everything': + self.grouping = history_grouping + + def prepare_container_image_old(self): self.docker.initdir() self.docker.pull_container_image(self.container_image_name) result = self.docker.get_container_image_history(self.container_image_name) From 11b27211c4099f8df6da3256e534146c91e9a210 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 29 Feb 2024 12:46:21 -0500 Subject: [PATCH 073/146] Fixed emty layer count discrepancy --- examples/client/multi-image/scan_docker_image_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index 84adcff5..65703190 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -241,7 +241,7 @@ def __init__( self.extra_options = detect_options.split(" ") print ("<--{}-->".format(self.grouping)) - def prepare_container_image_old(self): + def prepare_container_image(self): self.docker.initdir() self.docker.pull_container_image(self.container_image_name) self.docker.save_container_image(self.container_image_name) From 209eaa1f08e6a87528be17ffd081462f7ac19453 Mon Sep 17 00:00:00 2001 From: mkumykov <38922450+mkumykov@users.noreply.github.com> Date: Fri, 22 Mar 2024 09:42:05 -0400 Subject: [PATCH 074/146] Update Authentication.py Updating yo a relative URL to comply with RFC-3986 --- blackduck/Authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blackduck/Authentication.py b/blackduck/Authentication.py index c0f1f7ec..e6b22ea1 100644 --- a/blackduck/Authentication.py +++ b/blackduck/Authentication.py @@ -56,7 +56,7 @@ def authenticate(self): logger.warning("ssl verification disabled, connection insecure. do NOT use verify=False in production!") response = self.session.post( - url="/api/tokens/authenticate", + url="api/tokens/authenticate", auth=NoAuth(), # temporarily strip authentication to avoid infinite recursion headers={"Authorization": f"token {self.access_token}"} ) From 4b24cde425c65e6bfd1ae063aee69f41ca2de8c0 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 25 Mar 2024 13:52:24 -0400 Subject: [PATCH 075/146] new project format spec file added --- .../multi-image/manage_project_structure.py | 96 ++++++++++++------- 1 file changed, 64 insertions(+), 32 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 3a734e18..3572a4a4 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -204,31 +204,53 @@ def add_component_to_version_bom(child_version, version): data = { 'component': child_version['_meta']['href']} return bd.session.post(url, json=data) +def process_excel_spec_file(wb): + ws = wb.active + project_list = [] + row_number = 0 + for row in ws.values: + row_number += 1 + if (row_number == 1 and + row[0] == 'Container Name' and + row[1] == 'Image ID' and + row[2] == 'Version' and + row[3] == 'Project Name'): + logging.info(f"File Format checks out (kind of)") + continue + elif row_number > 1: + project_list.append(f"{row[3]}:{row[0]}:{row[2]}") + else: + logging.error(f"Could not parse input file {args.subproject_spec_file}") + sys.exit(1) + return (project_list) + +def process_text_spec_file(args): + project_list = [] + prefix = args.string_to_put_in_front_of_subproject_name + if not prefix: + prefix = args.project_name + with open(args.subproject_spec_file, "r") as f: + lines = f.read().splitlines() + for line in lines: + image_name = line.split('/')[-1].split(':')[0] # Don't look at me, you wrote it! + sub_project_name = "_".join((prefix, image_name)) + spec_line = ":".join((sub_project_name, line)) + if "ciena.com" in spec_line: + project_list.append(spec_line) + return (project_list) + def get_child_spec_list(args): if args.subproject_list: return args.subproject_list.split(',') else: + # Excel and plaintext logging.info(f"Processing excel file {args.subproject_spec_file}") import openpyxl - wb = openpyxl.load_workbook(args.subproject_spec_file) - ws = wb.active - project_list = [] - row_number = 0 - for row in ws.values: - row_number += 1 - if (row_number == 1 and - row[0] == 'Container Name' and - row[1] == 'Image ID' and - row[2] == 'Version' and - row[3] == 'Project Name'): - logging.info(f"File Format checks out (kind of)") - continue - elif row_number > 1: - project_list.append(f"{row[3]}:{row[0]}:{row[2]}") - else: - logging.error(f"Could not parse input file {args.subproject_spec_file}") - sys.exit(1) - return (project_list) + try: + wb = openpyxl.load_workbook(args.subproject_spec_file) + return process_excel_spec_file(wb) + except Exception: + return process_text_spec_file(args) def create_and_add_child_projects(version, args): version_url = version['_meta']['href'] + '/components' @@ -321,16 +343,20 @@ def scan_container_images(scan_params, hub): project_group = params.get('project_group', None) if project_group: detect_options += f" --detect.project.group.name=\"{project_group}\"" - scan_container_image( - params['image'], - None, - None, - None, - params['project'], - params['version'], - detect_options, - hub=hub - ) + try: + scan_container_image( + params['image'], + None, + None, + None, + params['project'], + params['version'], + detect_options, + hub=hub + ) + except Exception: + logging.error(f"Scanning of {params['image']} failed, skipping") + skipped_scans.append(params) def parse_command_args(): @@ -343,11 +369,12 @@ def parse_command_args(): parser.add_argument("-pv", "--version-name", required=True, help="Project Version Name") group = parser.add_mutually_exclusive_group() group.add_argument("-sp", "--subproject-list", required=False, help="List of subprojects to generate with subproject:container:tag") - group.add_argument("-ssf", "--subproject-spec-file", required=False, help="Excel file containing subproject specification") + group.add_argument("-ssf", "--subproject-spec-file", required=False, help="Excel or txt file containing subproject specification") parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") parser.add_argument("-rm", "--remove", action='store_true', required=False, help="Remove project structure with all subprojects (DANGEROUS!)") parser.add_argument("--clone-from", required=False, help="Main project version to use as template for cloning") parser.add_argument("--dry-run", action='store_true', required=False, help="Create structure only, do not execute scans") + parser.add_argument("-str", "--string-to-put-in-front-of-subproject-name", required=False, help="Prefix string for subproject names" ) return parser.parse_args() def main(): @@ -355,8 +382,9 @@ def main(): with open(args.token_file, 'r') as tf: access_token = tf.readline().strip() global bd - global scan_params + global scan_params, skipped_scans scan_params = [] + skipped_scans = [] bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) logging.info(f"{args}") @@ -367,10 +395,14 @@ def main(): if args.dry_run: logging.info(f"{pformat(scan_params)}") else: - logging.info("Now execution scans") + logging.info("Now executing scans") from blackduck.HubRestApi import HubInstance hub = HubInstance(args.base_url, api_token=access_token, insecure=True, debug=False) scan_container_images(scan_params, hub) + if len(skipped_scans) > 0: + logging.info(f"The following images were not scanned") + logging.info(f"{pformat(skipped_scans)}") + if __name__ == "__main__": From 83081e86a7342b660e9e4648e56a585db005bc7c Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 25 Mar 2024 13:55:40 -0400 Subject: [PATCH 076/146] new project format spec file added --- .../client/multi-image/manage_project_structure.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 3572a4a4..0aa48dff 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -34,7 +34,8 @@ Add on Image Version usage: python3 manage_project_structure.py [-h] -u BASE_URL -t TOKEN_FILE [-pg PROJECT_GROUP] -p PROJECT_NAME -pv VERSION_NAME - [-sp SUBPROJECT_LIST] [-nv] [-rm] [--clone-from CLONE_FROM] [--dry-run] + [-sp SUBPROJECT_LIST | -ssf SUBPROJECT_SPEC_FILE] [-nv] [-rm] [--clone-from CLONE_FROM] + [--dry-run] [-str STRING_TO_PUT_IN_FRONT_OF_SUBPROJECT_NAME] options: -h, --help show this help message and exit @@ -51,12 +52,13 @@ -sp SUBPROJECT_LIST, --subproject-list SUBPROJECT_LIST List of subprojects to generate with subproject:container:tag -ssf SUBPROJECT_SPEC_FILE, --subproject-spec-file SUBPROJECT_SPEC_FILE - Excel file containing subproject specification + Excel or txt file containing subproject specification -nv, --no-verify Disable TLS certificate verification -rm, --remove Remove project structure with all subprojects (DANGEROUS!) --clone-from CLONE_FROM Main project version to use as template for cloning --dry-run Create structure only, do not execute scans + -str STRING_TO_PUT_IN_FRONT_OF_SUBPROJECT_NAME, --string-to-put-in-front-of-subproject-name STRING_TO_PUT_IN_FRONT_OF_SUBPROJECT_NAME Subprojects ae specified as subproject:[container]:[tag] if container name omited it will be set to subproject @@ -67,6 +69,12 @@ Container Name, Image ID, Version, Project Name and subsequent rows containing data +Sublrojects could be specified in a text file with -ssf --subproject-spec-file parameter +Each line will have to contain full image specification. +Specification will be parsed and image name prefixed by -str parameter will be used as a subproject name +In this mode any image that is not residing on ciena.com repository will be skipped. +If -str parameter is empty, Project Name will be used instead. + Container image name scanned will be written into project version nickname field From 70180e26d3bee72b4574e52168e51a158d9d7657 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 1 Apr 2024 13:12:19 -0400 Subject: [PATCH 077/146] fixed SBOM mime type detection --- examples/client/upload_sbom.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/client/upload_sbom.py b/examples/client/upload_sbom.py index 931797da..89c1d1de 100644 --- a/examples/client/upload_sbom.py +++ b/examples/client/upload_sbom.py @@ -130,22 +130,22 @@ def find_or_create_project_version(project_name, version_name, project_group): sys.exit(1) def get_sbom_mime_type(filename): + import json with open(filename, 'r') as f: - data = f.readlines() - content = " ".join(data) - if 'CycloneDX' in content: + data = json.load(f) + if data.get('bomFormat', None) == "CycloneDX": return 'application/vnd.cyclonedx' - if 'SPDX' in content: + elif data.get('spdxVersion', None): return 'application/spdx' return None def upload_sbom_file(filename, project, version, project_group): find_or_create_project_version(project, version, project_group) mime_type = get_sbom_mime_type(filename) - print (mime_type) if not mime_type: logging.error(f"Could not identify file content for {filename}") sys.exit(1) + logging.info(f"Mime type {mime_type} will be used for file {filename}") files = {"file": (filename, open(filename,"rb"), mime_type)} fields = {"projectName": project, "versionName": version} response = bd.session.post("/api/scan/data", files = files, data=fields) From 4dde4411158b0321ebabae416987e797eba5f56d Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 2 Apr 2024 17:27:01 -0400 Subject: [PATCH 078/146] script to convert BOM to flat SBOM added --- examples/client/sbomify.py | 255 +++++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 examples/client/sbomify.py diff --git a/examples/client/sbomify.py b/examples/client/sbomify.py new file mode 100644 index 00000000..ffdd2deb --- /dev/null +++ b/examples/client/sbomify.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +''' +Created: Apr 2, 2024 +Author: @kumykov + +Copyright (c) 2024, Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +usage: sbomify.py [-h] -u BASE_URL -t TOKEN_FILE [-nv] -sp SOURCE_PROJECT -sv SOURCE_VERSION -tp TARGET_PROJECT -tv TARGET_VERSION + [-tg TARGET_PROJECT_GROUP] [-c] [--sbom-type SBOM_TYPE] + +Generate and download SBOM for the source project version +and upload it to the target project version + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -nv, --no-verify Disable TLS certificate verification + -sp SOURCE_PROJECT, --source-project SOURCE_PROJECT + Source Project Name + -sv SOURCE_VERSION, --source-version SOURCE_VERSION + Source Project Version Name + -tp TARGET_PROJECT, --target-project TARGET_PROJECT + Target Project Name + -tv TARGET_VERSION, --target-version TARGET_VERSION + Target Project Version Name + -tg TARGET_PROJECT_GROUP, --target-project-group TARGET_PROJECT_GROUP + Project Group to use for target + -c, --create-target Create target project version if does not exist + --sbom-type {SPDX_22,SPDX_23,CYCLONEDX_13,CYCLONEDX_14} + SBOM type to use for transaction + +Black Duck examples collection + + +''' +import argparse +import io +import json +import sys +import logging +import time + +from zipfile import ZipFile +from blackduck import Client + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + + +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'].casefold() == project_name.casefold()] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + +def find_or_create_project_group(bd, group_name): + url = '/api/project-groups' + params = { + 'q': [f"name:{group_name}"] + } + groups = [p for p in bd.get_items(url, params=params) if p['name'] == group_name] + if len(groups) == 0: + headers = { + 'Accept': 'application/vnd.blackducksoftware.project-detail-5+json', + 'Content-Type': 'application/vnd.blackducksoftware.project-detail-5+json' + } + data = { + 'name': group_name + } + response = bd.session.post(url, headers=headers, json=data) + return response.headers['Location'] + else: + return groups[0]['_meta']['href'] + +def create_project_version(bd, project_name,version_name,project_group, nickname = None): + version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name} + if nickname: + version_data['nickname'] = nickname + url = '/api/projects' + project = find_project_by_name(bd, project_name) + if project: + data = version_data + url = project['_meta']['href'] + '/versions' + else: + data = {"name": project_name, + "projectGroup": find_or_create_project_group(bd, project_group), + "versionRequest": version_data} + return bd.session.post(url, json=data) + +def locate_project_version(bd, project_name, version_name, group="Black Duck Project Groups", create=False): + project = find_project_by_name(bd, project_name) + version = None + if project: + version = find_project_version_by_name(bd, project, version_name) + if version: + pass + elif create: + version = create_project_version(bd, project_name, version_name, group) + else: + pass + elif create: + response = create_project_version(bd, project_name, version_name, group) + logging.info(f"Project {project_name} : {version_name} creation completed with {response}") + if response.ok: + project = find_project_by_name(bd, project_name) + version = find_project_version_by_name(bd, project, version_name) + return version + +def create_sbom_report(bd, version, type, include_subprojects): + post_data = { + 'reportFormat': "JSON", + 'sbomType': type, + 'includeSubprojects': include_subprojects + } + sbom_reports_url = version['_meta']['href'] + "/sbom-reports" + + bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json" + r = bd.session.post(sbom_reports_url, json=post_data) + if (r.status_code == 403): + logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") + r.raise_for_status() + location = r.headers.get('Location') + assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" + return location + +def download_report(bd, location, retries): + report_id = location.split("/")[-1] + if retries: + logging.debug(f"Retrieving generated report from {location}") + response = bd.session.get(location) + report_status = response.json().get('status', 'Not Ready') + if response.status_code == 200 and report_status == 'COMPLETED': + response = bd.session.get(location + "/download.zip", headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + if response.status_code == 200: + return response.content + else: + logging.error("Ruh-roh, not sure what happened here") + return None + else: + logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {retries} seconds then retrying...") + time.sleep(60) + retries -= 1 + return download_report(bd, location, retries) + else: + logging.debug(f"Failed to retrieve report {report_id} after multiple retries") + return None + +def produce_online_sbom_report(bd, project_name, project_version_name, sbom_type): + project = find_project_by_name(bd, project_name) + logging.debug(f"Project {project['name']} located") + version = find_project_version_by_name(bd, project, project_version_name) + logging.debug(f"Version {version['versionName']} located") + location = create_sbom_report(bd, version, sbom_type, True) + logging.debug(f"Created SBOM report of type {sbom_type} for project {project_name}, version {project_version_name} at location {location}") + sbom_data_zip = download_report(bd, location, 60) + logging.debug(f"Deleting report from Black Duck {bd.session.delete(location)}") + zip=ZipFile(io.BytesIO(sbom_data_zip), "r") + sbom_data = {name: zip.read(name) for name in zip.namelist()} + filename = [i for i in sbom_data.keys() if i.endswith(".json")][0] + return json.loads(sbom_data[filename]) + +def upload_sbom_file(bd, project_name, version_name, sbom_data): + if sbom_data.get('bomFormat', None) == "CycloneDX": + mime_type = 'application/vnd.cyclonedx' + elif sbom_data.get('spdxVersion', None): + mime_type = 'application/spdx' + else: + mime_type = None + if not mime_type: + logging.error(f"Could not identify file content for SBOM") + sys.exit(1) + logging.info(f"Mime type {mime_type} will be used for SBOM upload") + files = {"file": ('sbom.json', json.dumps(sbom_data).encode('utf-8'), mime_type)} + fields = {"projectName": project_name, "versionName": version_name} + response = bd.session.post("/api/scan/data", files = files, data=fields) + logging.info(f"SBOM Upload completed with {response}") + if response.status_code == 409: + logging.info(f"File SBOM is already mapped to a different project version") + +def sbomify(bd, args): + source = locate_project_version(bd, args.source_project, args.source_version) + if not source: + logging.error(f"Source project {args.source_project} : {args.source_version} not found. Exiting.") + sys.exit(1) + logging.info(f"Located source project {args.source_project} : {args.source_version}") + sbom = produce_online_sbom_report(bd, args.source_project, args.source_version, args.sbom_type) + bd.session.headers.pop('Content-Type') + target = locate_project_version(bd, args.target_project, args.target_version, group=args.target_project_group, create=args.create_target) + if not target: + logging.error(f"Target project {args.target_project} : {args.target_version} not found. Exiting.") + sys.exit(1) + logging.info(f"Located target project {args.target_project} : {args.target_version}") + upload_sbom_file(bd, args.target_project, args.target_version, sbom) + +def parse_command_args(): + parser = argparse.ArgumentParser(prog = "sbomify.py", description="Generate and download SBOM and upload to the target project version", epilog="Blackduck examples collection") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-sp", "--source-project", required=True, help="Source Project Name") + parser.add_argument("-sv", "--source-version", required=True, help="Source Project Version Name") + parser.add_argument("-tp", "--target-project", required=True, help="Target Project Name") + parser.add_argument("-tv", "--target-version", required=True, help="Target Project Version Name") + parser.add_argument("-tg", "--target-project-group", required=False, default='Black Duck Project Groups', help="Project Group to use for target") + parser.add_argument("-c", "--create-target", action='store_true', help="Create target project version if does not exist") + parser.add_argument("--sbom-type", required=False, default='SPDX_23', choices=["SPDX_22", "SPDX_23", "CYCLONEDX_13", "CYCLONEDX_14"], help="SBOM type to use for transaction") + + return parser.parse_args() + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + sbomify(bd, args) + +if __name__ == "__main__": + sys.exit(main()) From 13918ea8094e60d226da58837be5264cefe83a8c Mon Sep 17 00:00:00 2001 From: Andrew Calder Date: Fri, 19 Apr 2024 21:49:24 +0000 Subject: [PATCH 079/146] bump release to match pypi (dependency updates) --- blackduck/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blackduck/__version__.py b/blackduck/__version__.py index 4eaaf1f8..4d580b10 100644 --- a/blackduck/__version__.py +++ b/blackduck/__version__.py @@ -1,3 +1,3 @@ -VERSION = (1, 1, 0) +VERSION = (1, 1, 3) __version__ = '.'.join(map(str, VERSION)) From 662903b4a455f348816d91a1977cc10c80c4e825 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 3 May 2024 12:43:20 -0400 Subject: [PATCH 080/146] get project data --- examples/client/get_project_data.py | 111 ++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 examples/client/get_project_data.py diff --git a/examples/client/get_project_data.py b/examples/client/get_project_data.py new file mode 100644 index 00000000..89bf61c8 --- /dev/null +++ b/examples/client/get_project_data.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +''' +Created: Apr 2, 2024 +Author: @kumykov + +Copyright (c) 2024, Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +usage: get_project_data.py [-h] -u BASE_URL -t TOKEN_FILE [-nv] -p SOURCE_PROJECT -v SOURCE_VERSION + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -nv, --no-verify Disable TLS certificate verification + -p SOURCE_PROJECT, --project SOURCE_PROJECT + Project Name + -v SOURCE_VERSION, --version SOURCE_VERSION + Project Version Name + +Black Duck examples collection + + +''' +import argparse +import io +import json +import sys +import logging +import time + +from blackduck import Client +from pprint import pprint + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + + +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'].casefold() == project_name.casefold()] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + +def get_project_data(bd, args): + project = find_project_by_name(bd, args.project) + version = find_project_version_by_name(bd, project, args.version) + if not version: + logging.error(f"Source project {args.project} : {args.version} not found. Exiting.") + sys.exit(1) + logging.info(f"Located source project {args.project} : {args.version}") + return bd.get_resource('components', version) + + +def parse_command_args(): + parser = argparse.ArgumentParser(prog = "get_project_data.py", description="Generate and download SBOM and upload to the target project version", epilog="Blackduck examples collection") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-p", "--project", required=True, help="Project Name") + parser.add_argument("-v", "--version", required=True, help="Project Version Name") + + return parser.parse_args() + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + components = get_project_data(bd, args) + for component in components: + # pprint (component) + print (f"{component['componentName']} {component['componentVersionName']} {component['licenses'][0]['licenseDisplay']}") + +if __name__ == "__main__": + sys.exit(main()) From 4f4428a46cf3b6904aed128e6b3e03e0acd5bcae Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Sat, 22 Jun 2024 14:29:20 -0400 Subject: [PATCH 081/146] snippet matching example --- examples/client/match_snippet.py | 104 +++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 examples/client/match_snippet.py diff --git a/examples/client/match_snippet.py b/examples/client/match_snippet.py new file mode 100644 index 00000000..969bce89 --- /dev/null +++ b/examples/client/match_snippet.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +''' +Copyright (C) 2024 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +usage: match_snippets.py [-h] --base-url BASE_URL --token-file TOKEN_FILE [--no-verify] [--input INPUT] + +options: + -h, --help show this help message and exit + --base-url BASE_URL Hub server URL e.g. https://your.blackduck.url + --token-file TOKEN_FILE + containing access token + --no-verify disable TLS certificate verification + --input INPUT File containing code snippet or stdin + +Match a snippet of a code. +This functionality requires 'Generative AI Compliance' option licenses + + +Examples: + +Curl file content from github and match it against Black Duck KB +and format the output using jq utility + curl https://raw.githubusercontent.com/apache/kafka/trunk/shell/src/main/java/org/apache/kafka/shell/state/MetadataShellState.java | \ + python3 examples/client/match_snippet.py --base-url=$BD_URL --token-file=<(echo $API_TOKEN) --no-verify | \ + jq . + +This will produce something like: +{ + "snippetMatches": { + "PERMISSIVE": [ + { + "projectName": "Apache Kafka", + "releaseVersion": "3.5.0", + "licenseDefinition": { + "name": "Apache License 2.0", + "spdxId": "Apache-2.0", + "ownership": "OPEN_SOURCE", + "licenseDisplayName": "Apache License 2.0" +. . . + +''' +import argparse +import json +import logging +import sys + +from blackduck import Client + +parser = argparse.ArgumentParser('match_snippets.py') +parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token") +parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("--input", required=False, help="File containing code snippet or stdin") +args = parser.parse_args() + + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client( + base_url=args.base_url, + token=access_token, + verify=args.verify +) + +if args.input: + with open(args.input, 'r') as content_file: + content = content_file.read() +else: + with sys.stdin as content_file: + content = content_file.read() + +endpoint='/api/snippet-matching' +headers = {"Content-Type": "text/plain"} + +response = bd.session.post(url=endpoint, headers=headers, data=content) +if response.ok: + data = response.json() + import json + print(json.dumps(data)) \ No newline at end of file From 4e352145e30ccfccc00476a29f23598023b4c7ee Mon Sep 17 00:00:00 2001 From: dnichol Date: Tue, 25 Jun 2024 14:53:53 +0100 Subject: [PATCH 082/146] Hierarchy source report - basic structure and report generation --- examples/client/file_hierarchy_report.py | 265 +++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 examples/client/file_hierarchy_report.py diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py new file mode 100644 index 00000000..f756f3f6 --- /dev/null +++ b/examples/client/file_hierarchy_report.py @@ -0,0 +1,265 @@ +''' +Created on June 25, 2024 + +@author: dnichol and kumykov + +Generate version detail reports (source and components) and consolidate information on source matches, with license +and component matched. Removes matches found underneith other matched components in the source tree (configurable). + +Copyright (C) 2023 Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +''' + +import argparse +import logging +import sys +import os +import re +import time +import subprocess +import json +import traceback +import copy +import ijson +from blackduck import Client +from zipfile import ZipFile + +program_description = \ +'''Generate version detail reports (source and components) and consolidate information on source matches, with license +and component matched. Removes matches found underneith other matched components in the source tree (configurable). + +This script assumes a project version exists and has scans associated with it (i.e. the project is not scanned as part of this process). + +Config file: +API Token and Black Duck URL need to be placed in the .restconfig.json file which must be placed in the same folder where this script resides. + { + "baseurl": "https://hub-hostname", + "api_token": "", + "insecure": true or false , + "debug": true or false + } + +Remarks: +This script uses 3rd party PyPI package "ijson". This package must be installed. +''' + +# BD report general +BLACKDUCK_REPORT_MEDIATYPE = "application/vnd.blackducksoftware.report-4+json" +blackduck_report_download_api = "/api/projects/{projectId}/versions/{projectVersionId}/reports/{reportId}/download" +# BD version details report +blackduck_create_version_report_api = "/api/versions/{projectVersionId}/reports" +blackduck_version_report_filename = "./blackduck_version_report_for_{projectVersionId}.zip" +# Consolidated report +BLACKDUCK_VERSION_MEDIATYPE = "application/vnd.blackducksoftware.status-4+json" +BLACKDUCK_VERSION_API = "/api/current-version" +REPORT_DIR = "./blackduck_component_source_report" +# Retries to wait for BD report creation. RETRY_LIMIT can be overwritten by the script parameter. +RETRY_LIMIT = 30 +RETRY_TIMER = 30 + +def log_config(debug): + if debug: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.DEBUG) + else: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.INFO) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("blackduck").setLevel(logging.WARNING) + +def parse_parameter(): + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("project", + metavar="project", + type=str, + help="Provide the BlackDuck project name.") + parser.add_argument("version", + metavar="version", + type=str, + help="Provide the BlackDuck project version name.") + parser.add_argument("-kh", + "--keep_hierarchy", + action='store_true', + help="Set to keep all entries in the sources report. Will not remove components found under others.") + parser.add_argument("-rr", + "--report_retries", + metavar="", + type=int, + default=RETRY_LIMIT, + help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") + parser.add_argument("-t", + "--timeout", + metavar="", + type=int, + default=15, + help="Timeout for REST-API. Some API may take longer than the default 15 seconds") + parser.add_argument("-r", + "--retries", + metavar="", + type=int, + default=3, + help="Retries for REST-API. Some API may need more retries than the default 3 times") + return parser.parse_args() + +def get_bd_project_data(hub_client, project_name, version_name): + """ Get and return project ID, version ID. """ + project_id = "" + for project in hub_client.get_resource("projects"): + if project['name'] == project_name: + project_id = (project['_meta']['href']).split("projects/", 1)[1] + break + if project_id == "": + sys.exit(f"No project for {project_name} was found!") + version_id = codelocations = "" + for version in hub_client.get_resource("versions", project): + if version['versionName'] == version_name: + version_id = (version['_meta']['href']).split("versions/", 1)[1] + break + if version_id == "": + sys.exit(f"No project version for {version_name} was found!") + + return project_id, version_id + +def report_create(hub_client, url, body): + """ + Request BlackDuck to create report. Requested report is included in the request payload. + """ + res = hub_client.session.post(url, headers={'Content-Type': BLACKDUCK_REPORT_MEDIATYPE}, json=body) + if res.status_code != 201: + sys.exit(f"BlackDuck report creation failed with status {res.status_code}!") + return res.headers['Location'] # return report_url + +def report_download(hub_client, report_url, project_id, version_id, retries): + """ + Download the generated report after the report completion. We will retry until reaching the retry-limit. + """ + while retries: + res = hub_client.session.get(report_url, headers={'Accept': BLACKDUCK_REPORT_MEDIATYPE}) + if res.status_code == 200 and (json.loads(res.content))['status'] == "COMPLETED": + report_id = report_url.split("reports/", 1)[1] + download_url = (((blackduck_report_download_api.replace("{projectId}", project_id)) + .replace("{projectVersionId}", version_id)) + .replace("{reportId}", report_id)) + res = hub_client.session.get(download_url, + headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + if res.status_code != 200: + sys.exit(f"BlackDuck report download failed with status {res.status_code} for {download_url}!") + return res.content + elif res.status_code != 200: + sys.exit(f"BlackDuck report creation not completed successfully with status {res.status_code}") + else: + retries -= 1 + logging.info(f"Waiting for the report generation for {report_url} with the remaining retries {retries} times.") + time.sleep(RETRY_TIMER) + sys.exit(f"BlackDuck report for {report_url} was not generated after retries {RETRY_TIMER} sec * {retries} times!") + +def get_version_detail_report(hub_client, project_id, version_id, retries): + """ Create and get BOM component and BOM source file report in json. """ + create_version_url = blackduck_create_version_report_api.replace("{projectVersionId}", version_id) + body = { + 'reportFormat' : 'JSON', + 'locale' : 'en_US', + 'versionId' : f'{version_id}', + 'categories' : [ 'COMPONENTS', 'FILES' ] # Generating "project version" report including components and files + } + report_url = report_create(hub_client, create_version_url, body) + # Zipped report content is received and write the content to a local zip file + content = report_download(hub_client, report_url, project_id, version_id, retries) + output_file = blackduck_version_report_filename.replace("{projectVersionId}", version_id) + with open(output_file, "wb") as f: + f.write(content) + return output_file + +def get_blackduck_version(hub_client): + url = hub_client.base_url + BLACKDUCK_VERSION_API + res = hub_client.session.get(url) + if res.status_code == 200 and res.content: + return json.loads(res.content)['version'] + else: + sys.exit(f"Get BlackDuck version failed with status {res.status_code}") + +def generate_file_report(hub_client, project_id, version_id, keep_hierarchy, retries): + """ + Create a consolidated file report from BlackDuck project version source and components reports. + Remarks: + """ + if not os.path.exists(REPORT_DIR): + os.makedirs(REPORT_DIR) + + # Report body - Component BOM, file BOM with Discoveries data + version_report_zip = get_version_detail_report(hub_client, project_id, version_id, retries) + with ZipFile(f"./{version_report_zip}", "r") as vzf: + vzf.extractall() + for i, unzipped_version in enumerate(vzf.namelist()): + if re.search(r"\bversion.+json\b", unzipped_version) is not None: + break + if i + 1 >= len(vzf.namelist()): + sys.exit(f"Version detail file not found in the downloaded report: {version_report_zip}!") + + # Report body - Component BOM report + with open(f"./{unzipped_version}", "r") as uvf: + for i, comp_bom in enumerate(ijson.items(uvf, 'aggregateBomViewEntries.item')): + logging.info(f"{comp_bom['componentName']}") + logging.info(f"Number of the reported components {i+1}") + + +def main(): + args = parse_parameter() + debug = 0 + try: + if args.project == "": + sys.exit("Please set BlackDuck project name!") + if args.version == "": + sys.exit("Please set BlackDuck project version name!") + + with open(".restconfig.json", "r") as f: + config = json.load(f) + # Remove last slash if there is, otherwise REST API may fail. + if re.search(r".+/$", config['baseurl']): + bd_url = config['baseurl'][:-1] + else: + bd_url = config['baseurl'] + bd_token = config['api_token'] + bd_insecure = not config['insecure'] + if config['debug']: + debug = 1 + + log_config(debug) + + hub_client = Client(token=bd_token, + base_url=bd_url, + verify=bd_insecure, + timeout=args.timeout, + retries=args.retries) + + project_id, version_id = get_bd_project_data(hub_client, args.project, args.version) + + generate_file_report(hub_client, + project_id, + version_id, + args.keep_hierarchy, + args.report_retries + ) + + except (Exception, BaseException) as err: + logging.error(f"Exception by {str(err)}. See the stack trace") + traceback.print_exc() + +if __name__ == '__main__': + sys.exit(main()) From bd3620970c5b750034d11dddfe7cedaf54fee36f Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 2 Jul 2024 20:44:20 -0400 Subject: [PATCH 083/146] added OCI format to scan_docker_image_lite --- examples/scan_docker_image_lite.py | 119 ++++++++++++++++++++++++----- 1 file changed, 99 insertions(+), 20 deletions(-) diff --git a/examples/scan_docker_image_lite.py b/examples/scan_docker_image_lite.py index e3c41cef..46f1027e 100644 --- a/examples/scan_docker_image_lite.py +++ b/examples/scan_docker_image_lite.py @@ -5,7 +5,7 @@ Alternative version if Docker image layer by layer scan. -This program will download docker image and scan it into Blackduck server layer by layer +This program will download docker image and scan it into Black Duck server layer by layer Each layer will be scanned as a separate scan with a signature scan. Layers in the container images could be grouped into groups of contiguous layers. @@ -50,16 +50,16 @@ optional arguments: -h, --help show this help message and exit - --grouping GROUPING Group layers into user defined provect versions (can't be used with --base-image) + --grouping GROUPING Group layers into user defined project versions (can't be used with --base-image) --base-image BASE_IMAGE Use base image spec to determine base image/layers (can't be used with --grouping or --dockerfile) --dockerfile DOCKERFILE Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image) --project-name Specify project name (default is container image spec) - --project-verson Specify project version (default is container image tag/version) + --project-version Specify project version (default is container image tag/version) --detect-options DETECT_OPTIONS - Extra detect options to be passed directlyto the detect + Extra detect options to be passed directly to the detect Using --detect-options @@ -188,12 +188,24 @@ def read_config(self): with open(configFile) as fp: data = json.load(fp) return data + + def read_oci_layout(self): + oci_layout_file = self.imagedir + "/" + 'oci-layout' + if os.path.exists(oci_layout_file) and os.path.isfile(oci_layout_file): + with open(oci_layout_file) as fp: + data = json.load(fp) + return data + else: + return None + class Detector(): def __init__(self, hub): # self.detecturl = 'https://blackducksoftware.github.io/hub-detect/hub-detect.sh' # self.detecturl = 'https://detect.synopsys.com/detect.sh' - self.detecturl = 'https://detect.synopsys.com/detect7.sh' + # self.detecturl = 'https://detect.synopsys.com/detect7.sh' + # self.detecturl = 'https://detect.synopsys.com/detect8.sh' + self.detecturl = 'https://detect.synopsys.com/detect9.sh' self.baseurl = hub.config['baseurl'] self.filename = '/tmp/hub-detect.sh' self.token=hub.config['api_token'] @@ -240,6 +252,7 @@ def __init__( if detect_options: self.extra_options = detect_options.split(" ") print ("<--{}-->".format(self.grouping)) + self.binary = False def prepare_container_image(self): self.docker.initdir() @@ -262,6 +275,7 @@ def prepare_container_image(self): self.grouping = history_grouping self.docker.save_container_image(self.container_image_name) self.docker.unravel_container() + self.oci_layout = self.docker.read_oci_layout() def process_container_image_by_user_defined_groups(self): self.manifest = self.docker.read_manifest() @@ -333,25 +347,86 @@ def process_container_image_by_base_image_info(self): num = num + 1 print (json.dumps(self.layers, indent=4)) + def process_oci_container_image_by_user_defined_groups(self): + self.manifest = self.docker.read_manifest() + self.config = self.docker.read_config() + + self.layers = self.config['history'] + tagged_layers = [x for x in self.layers if '_group_end' in x.get('created_by')] + groups = {re.search('echo (.+?)_group_end', str(x['created_by'])).group(1): self.layers.index(x) for x in tagged_layers} + print (groups) + layer_paths = self.manifest[0]['Layers'].copy() + empty_layers = [x for x in self.layers if x.get('empty_layer', False)] + print(f"Total layers: {len(self.layers)} total paths: {len(layer_paths)} empty layers: {len(empty_layers)}") + + assert len(self.layers) == len(layer_paths) + len(empty_layers), "Something is wrong with this image, Layer math does not add up." + + for layer in self.layers: + layer['index'] = self.layers.index(layer) + if self.grouping: + layer['group_name'] = self.get_group_name(groups, layer['index']) + layer['project_version'] = "{}_{}".format(self.project_version,layer['group_name']) + layer['name'] = "{}_{}_{}_layer_{}".format(self.project_name,self.project_version,layer['group_name'],str(layer['index'])) + else: + layer['project_version'] = self.project_version + layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(layer['index']) + layer['project_name'] = self.project_name + if not layer.get('empty_layer', False): + layer['path'] = layer_paths.pop(0) + print (json.dumps(self.layers, indent=4)) + + def get_group_name(self, groups, index): + group_name = 'undefined' + for group, value in groups.items(): + if index <= value: + group_name = group + break + return group_name + + def process_oci_container_image_by_base_image_info(self): + print ("Processing by BAse Image not supported for OCI images") + sys.exit(1) + pass + def process_container_image(self): + if self.oci_layout: + self.process_oci_container_image() + else: + self.process_docker_container_image() + + def process_docker_container_image(self): if self.grouping: self.process_container_image_by_user_defined_groups() else: self.process_container_image_by_base_image_info() + def process_oci_container_image(self): + if self.grouping: + self.process_oci_container_image_by_user_defined_groups() + else: + self.process_oci_container_image_by_base_image_info() + def submit_layer_scans(self): for layer in self.layers: - options = [] - options.append('--detect.project.name={}'.format(layer['project_name'])) - options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) - # options.append('--detect.blackduck.signature.scanner.disabled=false') - options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) - options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) - if self.base_image or self.grouping or self.dockerfile: - options.extend(self.adorn_extra_options(layer)) - else: - options.extend(self.extra_options) - self.hub_detect.detect_run(options) + if not layer.get('empty_layer', False): + options = [] + options.append('--detect.project.name={}'.format(layer['project_name'])) + options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) + options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) + if self.binary: + options.append('--detect.tools=BINARY_SCAN') + options.append('--detect.binary.scan.file.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.tools=SIGNATURE_SCAN') + if self.oci_layout: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) + if self.base_image or self.grouping or self.dockerfile: + options.extend(self.adorn_extra_options(layer)) + else: + options.extend(self.extra_options) + self.hub_detect.detect_run(options) def adorn_extra_options(self, layer): result = list() @@ -401,7 +476,7 @@ def get_base_layers(self): def scan_container_image( imagespec, grouping=None, base_image=None, dockerfile=None, - project_name=None, project_version=None, detect_options=None): + project_name=None, project_version=None, detect_options=None, binary=False): hub = HubInstance() scanner = ContainerImageScanner( @@ -416,6 +491,8 @@ def scan_container_image( scanner.grouping = '1024:everything' else: scanner.base_layers = scanner.get_base_layers() + if binary: + scanner.binary = True scanner.prepare_container_image() scanner.process_container_image() scanner.submit_layer_scans() @@ -429,12 +506,13 @@ def main(argv=None): parser = ArgumentParser() parser.add_argument('imagespec', help="Container image tag, e.g. repository/imagename:version") - parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined provect versions (can't be used with --base-image)") + parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined project versions (can't be used with --base-image)") parser.add_argument('--base-image',default=None, type=str, help="Use base image spec to determine base image/layers (can't be used with --grouping or --dockerfile)") parser.add_argument('--dockerfile',default=None, type=str, help="Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image)") parser.add_argument('--project-name',default=None, type=str, help="Specify project name (default is container image spec)") parser.add_argument('--project-version',default=None, type=str, help="Specify project version (default is container image tag/version)") - parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directlyto the detect") + parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directly to the detect") + parser.add_argument('--binary', action='store_true', help="Use Binary Scan instead of signature scan") args = parser.parse_args() @@ -459,7 +537,8 @@ def main(argv=None): args.dockerfile, args.project_name, args.project_version, - args.detect_options) + args.detect_options, + args.binary) if __name__ == "__main__": From a7ad4be1d386fa36291f2111aa6ef8da0a652732 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 9 Jul 2024 09:46:09 -0400 Subject: [PATCH 084/146] 0.5 prototype --- examples/client/file_hierarchy_report.py | 228 ++++++++++------------- 1 file changed, 94 insertions(+), 134 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index f756f3f6..0dba1657 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -30,6 +30,7 @@ import argparse import logging import sys +import io import os import re import time @@ -40,10 +41,11 @@ import ijson from blackduck import Client from zipfile import ZipFile +from pprint import pprint program_description = \ '''Generate version detail reports (source and components) and consolidate information on source matches, with license -and component matched. Removes matches found underneith other matched components in the source tree (configurable). +and component matched. Removes matches found underneath other matched components in the source tree (configurable). This script assumes a project version exists and has scans associated with it (i.e. the project is not scanned as part of this process). @@ -83,39 +85,21 @@ def log_config(debug): logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("blackduck").setLevel(logging.WARNING) -def parse_parameter(): - parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("project", - metavar="project", - type=str, - help="Provide the BlackDuck project name.") - parser.add_argument("version", - metavar="version", - type=str, - help="Provide the BlackDuck project version name.") - parser.add_argument("-kh", - "--keep_hierarchy", - action='store_true', - help="Set to keep all entries in the sources report. Will not remove components found under others.") - parser.add_argument("-rr", - "--report_retries", - metavar="", - type=int, - default=RETRY_LIMIT, - help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") - parser.add_argument("-t", - "--timeout", - metavar="", - type=int, - default=15, - help="Timeout for REST-API. Some API may take longer than the default 15 seconds") - parser.add_argument("-r", - "--retries", - metavar="", - type=int, - default=3, - help="Retries for REST-API. Some API may need more retries than the default 3 times") - return parser.parse_args() +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + assert len(projects) == 1, f"Project {project_name} not found." + return projects[0] + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + assert len(versions) == 1, f"Project version {version_name} for project {project['name']} not found" + return versions[0] def get_bd_project_data(hub_client, project_name, version_name): """ Get and return project ID, version ID. """ @@ -136,55 +120,53 @@ def get_bd_project_data(hub_client, project_name, version_name): return project_id, version_id -def report_create(hub_client, url, body): - """ - Request BlackDuck to create report. Requested report is included in the request payload. - """ - res = hub_client.session.post(url, headers={'Content-Type': BLACKDUCK_REPORT_MEDIATYPE}, json=body) - if res.status_code != 201: - sys.exit(f"BlackDuck report creation failed with status {res.status_code}!") - return res.headers['Location'] # return report_url - -def report_download(hub_client, report_url, project_id, version_id, retries): - """ - Download the generated report after the report completion. We will retry until reaching the retry-limit. - """ - while retries: - res = hub_client.session.get(report_url, headers={'Accept': BLACKDUCK_REPORT_MEDIATYPE}) - if res.status_code == 200 and (json.loads(res.content))['status'] == "COMPLETED": - report_id = report_url.split("reports/", 1)[1] - download_url = (((blackduck_report_download_api.replace("{projectId}", project_id)) - .replace("{projectVersionId}", version_id)) - .replace("{reportId}", report_id)) - res = hub_client.session.get(download_url, - headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) - if res.status_code != 200: - sys.exit(f"BlackDuck report download failed with status {res.status_code} for {download_url}!") - return res.content - elif res.status_code != 200: - sys.exit(f"BlackDuck report creation not completed successfully with status {res.status_code}") - else: - retries -= 1 - logging.info(f"Waiting for the report generation for {report_url} with the remaining retries {retries} times.") - time.sleep(RETRY_TIMER) - sys.exit(f"BlackDuck report for {report_url} was not generated after retries {RETRY_TIMER} sec * {retries} times!") - -def get_version_detail_report(hub_client, project_id, version_id, retries): - """ Create and get BOM component and BOM source file report in json. """ - create_version_url = blackduck_create_version_report_api.replace("{projectVersionId}", version_id) - body = { +def create_version_details_report(bd, version): + version_reports_url = bd.list_resources(version).get('versionReport') + post_data = { 'reportFormat' : 'JSON', 'locale' : 'en_US', - 'versionId' : f'{version_id}', + 'versionId': version['_meta']['href'].split("/")[-1], 'categories' : [ 'COMPONENTS', 'FILES' ] # Generating "project version" report including components and files } - report_url = report_create(hub_client, create_version_url, body) - # Zipped report content is received and write the content to a local zip file - content = report_download(hub_client, report_url, project_id, version_id, retries) - output_file = blackduck_version_report_filename.replace("{projectVersionId}", version_id) - with open(output_file, "wb") as f: - f.write(content) - return output_file + + bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json" + r = bd.session.post(version_reports_url, json=post_data) + if (r.status_code == 403): + logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") + r.raise_for_status() + pprint(r.headers) + location = r.headers.get('Location') + assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" + return location + +def download_report(bd, location, retries): + report_id = location.split("/")[-1] + print (location) + url_data = location.split('/') + url_data.pop(4) + url_data.pop(4) + download_link = '/'.join(url_data) + print(download_link) + if retries: + logging.debug(f"Retrieving generated report from {location}") + response = bd.session.get(location) + report_status = response.json().get('status', 'Not Ready') + if response.status_code == 200 and report_status == 'COMPLETED': + response = bd.session.get(download_link, headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + pprint(response) + if response.status_code == 200: + return response.content + else: + logging.error("Ruh-roh, not sure what happened here") + return None + else: + logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {retries} seconds then retrying...") + time.sleep(60) + retries -= 1 + return download_report(bd, location, retries) + else: + logging.debug(f"Failed to retrieve report {report_id} after multiple retries") + return None def get_blackduck_version(hub_client): url = hub_client.base_url + BLACKDUCK_VERSION_API @@ -194,68 +176,46 @@ def get_blackduck_version(hub_client): else: sys.exit(f"Get BlackDuck version failed with status {res.status_code}") -def generate_file_report(hub_client, project_id, version_id, keep_hierarchy, retries): - """ - Create a consolidated file report from BlackDuck project version source and components reports. - Remarks: - """ - if not os.path.exists(REPORT_DIR): - os.makedirs(REPORT_DIR) - - # Report body - Component BOM, file BOM with Discoveries data - version_report_zip = get_version_detail_report(hub_client, project_id, version_id, retries) - with ZipFile(f"./{version_report_zip}", "r") as vzf: - vzf.extractall() - for i, unzipped_version in enumerate(vzf.namelist()): - if re.search(r"\bversion.+json\b", unzipped_version) is not None: - break - if i + 1 >= len(vzf.namelist()): - sys.exit(f"Version detail file not found in the downloaded report: {version_report_zip}!") - - # Report body - Component BOM report - with open(f"./{unzipped_version}", "r") as uvf: - for i, comp_bom in enumerate(ijson.items(uvf, 'aggregateBomViewEntries.item')): - logging.info(f"{comp_bom['componentName']}") - logging.info(f"Number of the reported components {i+1}") - +def parse_command_args(): + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") + parser.add_argument("-pn", "--project-name", required=True, help="Project Name") + parser.add_argument("-pv", "--project-version-name", required=True, help="Project Version Name") + parser.add_argument("-kh", "--keep_hierarchy", action='store_true', help="Set to keep all entries in the sources report. Will not remove components found under others.") + parser.add_argument("--report-retries", metavar="", type=int, default=RETRY_LIMIT, help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") + parser.add_argument("--timeout", metavar="", type=int, default=60, help="Timeout for REST-API. Some API may take longer than the default 60 seconds") + parser.add_argument("--retries", metavar="", type=int, default=4, help="Retries for REST-API. Some API may need more retries than the default 4 times") + return parser.parse_args() def main(): - args = parse_parameter() - debug = 0 + args = parse_command_args() + with open(args.token_file, 'r') as tf: + token = tf.readline().strip() try: - if args.project == "": - sys.exit("Please set BlackDuck project name!") - if args.version == "": - sys.exit("Please set BlackDuck project version name!") - - with open(".restconfig.json", "r") as f: - config = json.load(f) - # Remove last slash if there is, otherwise REST API may fail. - if re.search(r".+/$", config['baseurl']): - bd_url = config['baseurl'][:-1] - else: - bd_url = config['baseurl'] - bd_token = config['api_token'] - bd_insecure = not config['insecure'] - if config['debug']: - debug = 1 - - log_config(debug) - - hub_client = Client(token=bd_token, - base_url=bd_url, - verify=bd_insecure, + log_config(args.debug) + hub_client = Client(token=token, + base_url=args.base_url, + verify=args.no_verify, timeout=args.timeout, retries=args.retries) - project_id, version_id = get_bd_project_data(hub_client, args.project, args.version) + project = find_project_by_name(hub_client, args.project_name) + version = find_project_version_by_name(hub_client, project, args.project_version_name) + pprint(version) + location = create_version_details_report(hub_client, version) + pprint(location) + report_zip = download_report(hub_client, location, args.report_retries) + pprint(report_zip) + logging.debug(f"Deleting report from Black Duck {hub_client.session.delete(location)}") + zip=ZipFile(io.BytesIO(report_zip), "r") + pprint(zip.namelist()) + report_data = {name: zip.read(name) for name in zip.namelist()} + filename = [i for i in report_data.keys() if i.endswith(".json")][0] + pprint(json.loads(report_data[filename])) - generate_file_report(hub_client, - project_id, - version_id, - args.keep_hierarchy, - args.report_retries - ) except (Exception, BaseException) as err: logging.error(f"Exception by {str(err)}. See the stack trace") From aa08801a84c8c2d91526850a61a3640f9832f7ce Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 9 Jul 2024 10:20:35 -0400 Subject: [PATCH 085/146] 0.5 prototype --- examples/client/file_hierarchy_report.py | 33 ++++++------------------ 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 0dba1657..d380dd9f 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -31,14 +31,9 @@ import logging import sys import io -import os -import re import time -import subprocess import json import traceback -import copy -import ijson from blackduck import Client from zipfile import ZipFile from pprint import pprint @@ -49,17 +44,6 @@ This script assumes a project version exists and has scans associated with it (i.e. the project is not scanned as part of this process). -Config file: -API Token and Black Duck URL need to be placed in the .restconfig.json file which must be placed in the same folder where this script resides. - { - "baseurl": "https://hub-hostname", - "api_token": "", - "insecure": true or false , - "debug": true or false - } - -Remarks: -This script uses 3rd party PyPI package "ijson". This package must be installed. ''' # BD report general @@ -134,26 +118,24 @@ def create_version_details_report(bd, version): if (r.status_code == 403): logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") r.raise_for_status() - pprint(r.headers) location = r.headers.get('Location') assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" return location def download_report(bd, location, retries): report_id = location.split("/")[-1] - print (location) + logging.debug(f"Report location {location}") url_data = location.split('/') url_data.pop(4) url_data.pop(4) download_link = '/'.join(url_data) - print(download_link) + logging.debug(f"Report Download link {download_link}") if retries: - logging.debug(f"Retrieving generated report from {location}") + logging.debug(f"Retrieving generated report for {location} via {download_link}") response = bd.session.get(location) report_status = response.json().get('status', 'Not Ready') if response.status_code == 200 and report_status == 'COMPLETED': response = bd.session.get(download_link, headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) - pprint(response) if response.status_code == 200: return response.content else: @@ -204,17 +186,18 @@ def main(): project = find_project_by_name(hub_client, args.project_name) version = find_project_version_by_name(hub_client, project, args.project_version_name) - pprint(version) location = create_version_details_report(hub_client, version) - pprint(location) report_zip = download_report(hub_client, location, args.report_retries) - pprint(report_zip) logging.debug(f"Deleting report from Black Duck {hub_client.session.delete(location)}") zip=ZipFile(io.BytesIO(report_zip), "r") pprint(zip.namelist()) report_data = {name: zip.read(name) for name in zip.namelist()} filename = [i for i in report_data.keys() if i.endswith(".json")][0] - pprint(json.loads(report_data[filename])) + version_report = json.loads(report_data[filename]) + # TODO items + # Process file section of report data to identify primary paths + # Combine component data with selected file data + # Output result with CSV anf JSON as options. except (Exception, BaseException) as err: From 96feab654c9b864176825b31eee9b7e9dbe2a86b Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 10 Jul 2024 13:34:55 -0400 Subject: [PATCH 086/146] . --- examples/client/file_hierarchy_report.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index d380dd9f..c86d06da 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -194,6 +194,8 @@ def main(): report_data = {name: zip.read(name) for name in zip.namelist()} filename = [i for i in report_data.keys() if i.endswith(".json")][0] version_report = json.loads(report_data[filename]) + with open("out.json", "w") as f: + json.dump(version_report, f) # TODO items # Process file section of report data to identify primary paths # Combine component data with selected file data From 46e66a6d7b8e1f0c875602fd747c2d565303ec2f Mon Sep 17 00:00:00 2001 From: Dinesh Ravi Date: Thu, 11 Jul 2024 18:38:35 +0200 Subject: [PATCH 087/146] Create remap_codelocations.py --- examples/client/remap_codelocations.py | 91 ++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 examples/client/remap_codelocations.py diff --git a/examples/client/remap_codelocations.py b/examples/client/remap_codelocations.py new file mode 100644 index 00000000..089a97bd --- /dev/null +++ b/examples/client/remap_codelocations.py @@ -0,0 +1,91 @@ +""" +Created on july 11, 2024 + +@author: Dinesh Ravi + +Remap codelocations from a project version to another project version + +""" + +from blackduck import Client + +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s", +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument( + "-u", "--bd_url", help="Hub server URL e.g. https://your.blackduck.url" +) +parser.add_argument( + "-t", "--token-file", help="File name of a file containing access token" +) +parser.add_argument( + "-nv", + "--no-verify", + dest="verify", + action="store_false", + help="disable TLS certificate verification", +) +parser.add_argument("project_name") +parser.add_argument("version_name") +parser.add_argument("update_pv_url") + + +args = parser.parse_args() + +logging.basicConfig( + format="%(asctime)s:%(levelname)s:%(message)s", + stream=sys.stderr, + level=logging.DEBUG, +) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, "r") as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) + +params = {"q": [f"name:{args.project_name}"]} +projects = [ + p + for p in bd.get_resource("projects", params=params) + if p["name"] == args.project_name +] +assert ( + len(projects) == 1 +), f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +params = {"q": [f"versionName:{args.version_name}"]} +versions = [ + v + for v in bd.get_resource("versions", project, params=params) + if v["versionName"] == args.version_name +] +assert ( + len(versions) == 1 +), f"There should be one, and only one version named {args.version_name}. We found {len(versions)}" +version = versions[0] + +logging.debug(f"Found {project['name']}:{version['versionName']}") + + +codelocations = bd.get_resource("codelocations", version) +# logging.info(f"Total Code locations '{len(list(codelocations))}'") +for codelocation in codelocations: + logging.debug(f"Un-mapping code location {codelocation['name']}") + url = codelocation["_meta"]["href"] + codelocation["mappedProjectVersion"] = args.update_pv_url + result = bd.session.put(url, json=codelocation) + logging.info(f"Code location '{codelocation['name']}' unmap status {result}") From 9b63b785822443039e3893fff91097d872d75b63 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 11 Jul 2024 15:57:04 -0400 Subject: [PATCH 088/146] Functional requirement satisfied --- examples/client/file_hierarchy_report.py | 78 ++++++++++++++++++++---- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index c86d06da..f97e96b4 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -28,6 +28,7 @@ ''' import argparse +import csv import logging import sys import io @@ -47,15 +48,8 @@ ''' # BD report general -BLACKDUCK_REPORT_MEDIATYPE = "application/vnd.blackducksoftware.report-4+json" -blackduck_report_download_api = "/api/projects/{projectId}/versions/{projectVersionId}/reports/{reportId}/download" -# BD version details report -blackduck_create_version_report_api = "/api/versions/{projectVersionId}/reports" -blackduck_version_report_filename = "./blackduck_version_report_for_{projectVersionId}.zip" -# Consolidated report BLACKDUCK_VERSION_MEDIATYPE = "application/vnd.blackducksoftware.status-4+json" BLACKDUCK_VERSION_API = "/api/current-version" -REPORT_DIR = "./blackduck_component_source_report" # Retries to wait for BD report creation. RETRY_LIMIT can be overwritten by the script parameter. RETRY_LIMIT = 30 RETRY_TIMER = 30 @@ -122,7 +116,7 @@ def create_version_details_report(bd, version): assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" return location -def download_report(bd, location, retries): +def download_report(bd, location, retries, timeout): report_id = location.split("/")[-1] logging.debug(f"Report location {location}") url_data = location.split('/') @@ -142,10 +136,10 @@ def download_report(bd, location, retries): logging.error("Ruh-roh, not sure what happened here") return None else: - logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {retries} seconds then retrying...") - time.sleep(60) + logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {timeout} seconds then retrying...") + time.sleep(timeout) retries -= 1 - return download_report(bd, location, retries) + return download_report(bd, location, retries, timeout) else: logging.debug(f"Failed to retrieve report {report_id} after multiple retries") return None @@ -158,6 +152,47 @@ def get_blackduck_version(hub_client): else: sys.exit(f"Get BlackDuck version failed with status {res.status_code}") +def reduce(path_set): + path_set.sort() + for path in path_set: + if len(path) < 3: + continue + index = path_set.index(path) + while index + 1 < len(path_set) and path in path_set[index+1]: + logging.debug(f"{path} is in {path_set[index+1]} deleting the sub-path from the list") + path_set.pop(index+1) + return path_set + +def trim_version_report(version_report, reduced_path_set): + file_bom_entries = version_report['detailedFileBomViewEntries'] + aggregate_bom_view_entries = version_report['aggregateBomViewEntries'] + + reduced_file_bom_entries = [e for e in file_bom_entries if f"{e.get('archiveContext', "")}!{e['path']}" in reduced_path_set] + version_report['detailedFileBomViewEntries'] = reduced_file_bom_entries + + component_identifiers = [f"{e['projectId']}:{e['versionId']}" for e in reduced_file_bom_entries] + deduplicated = list(dict.fromkeys(component_identifiers)) + + reduced_aggregate_bom_view_entries = [e for e in aggregate_bom_view_entries if f"{e['producerProject']['id']}:{e['producerReleases'][0]['id']}" in deduplicated] + version_report['aggregateBomViewEntries'] = reduced_aggregate_bom_view_entries + +def write_output_file(version_report, output_file): + if output_file.lower().endswith(".csv"): + logging.info(f"Writing CSV output into {output_file}") + field_names = list(version_report['aggregateBomViewEntries'][0].keys()) + with open(output_file, "w") as f: + writer = csv.DictWriter(f, fieldnames = field_names) + writer.writeheader() + writer.writerows(version_report['aggregateBomViewEntries']) + + return + # If it's neither, then .json + if not output_file.lower().endswith(".json"): + output_file += ".json" + logging.info(f"Writing JSON output into {output_file}") + with open(output_file,"w") as f: + json.dump(version_report, f) + def parse_command_args(): parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") @@ -166,8 +201,10 @@ def parse_command_args(): parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") parser.add_argument("-pn", "--project-name", required=True, help="Project Name") parser.add_argument("-pv", "--project-version-name", required=True, help="Project Version Name") + parser.add_argument("-o", "--output-file", required=False, help="File name to write output. File extension determines format .json and .csv, json is the default.") parser.add_argument("-kh", "--keep_hierarchy", action='store_true', help="Set to keep all entries in the sources report. Will not remove components found under others.") parser.add_argument("--report-retries", metavar="", type=int, default=RETRY_LIMIT, help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") + parser.add_argument("--report-timeout", metavar="", type=int, default=RETRY_TIMER, help="Wait time between subsequent download attempts.") parser.add_argument("--timeout", metavar="", type=int, default=60, help="Timeout for REST-API. Some API may take longer than the default 60 seconds") parser.add_argument("--retries", metavar="", type=int, default=4, help="Retries for REST-API. Some API may need more retries than the default 4 times") return parser.parse_args() @@ -176,6 +213,9 @@ def main(): args = parse_command_args() with open(args.token_file, 'r') as tf: token = tf.readline().strip() + output_file = args.output_file + if not args.output_file: + output_file = f"{args.project_name}-{args.project_version_name}.json".replace(" ","_") try: log_config(args.debug) hub_client = Client(token=token, @@ -187,7 +227,7 @@ def main(): project = find_project_by_name(hub_client, args.project_name) version = find_project_version_by_name(hub_client, project, args.project_version_name) location = create_version_details_report(hub_client, version) - report_zip = download_report(hub_client, location, args.report_retries) + report_zip = download_report(hub_client, location, args.report_retries, args.report_timeout) logging.debug(f"Deleting report from Black Duck {hub_client.session.delete(location)}") zip=ZipFile(io.BytesIO(report_zip), "r") pprint(zip.namelist()) @@ -198,10 +238,24 @@ def main(): json.dump(version_report, f) # TODO items # Process file section of report data to identify primary paths + path_set = [f"{entry.get('archiveContext', "")}!{entry['path']}" for entry in version_report['detailedFileBomViewEntries']] + reduced_path_set = reduce(path_set.copy()) + logging.info(f"{len(path_set)-len(reduced_path_set)} path entries were scrubbed from the dataset.") + + # Remove component entries that correspond to removed path entries. + + logging.info(f"Original dataset contains {len(version_report['aggregateBomViewEntries'])} bom entries and {len(version_report['detailedFileBomViewEntries'])} file view entries") + if not args.keep_hierarchy: + trim_version_report(version_report, reduced_path_set) + logging.info(f"Truncated dataset contains {len(version_report['aggregateBomViewEntries'])} bom entries and {len(version_report['detailedFileBomViewEntries'])} file view entries") + + write_output_file(version_report, output_file) + # Combine component data with selected file data # Output result with CSV anf JSON as options. + except (Exception, BaseException) as err: logging.error(f"Exception by {str(err)}. See the stack trace") traceback.print_exc() From 5d2ac5d4b1239f630d7ff438c091409a83534776 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 12 Jul 2024 09:23:08 -0400 Subject: [PATCH 089/146] remove double quotes inside f-interpolated string --- examples/client/file_hierarchy_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index f97e96b4..74c1cdb4 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -167,7 +167,7 @@ def trim_version_report(version_report, reduced_path_set): file_bom_entries = version_report['detailedFileBomViewEntries'] aggregate_bom_view_entries = version_report['aggregateBomViewEntries'] - reduced_file_bom_entries = [e for e in file_bom_entries if f"{e.get('archiveContext', "")}!{e['path']}" in reduced_path_set] + reduced_file_bom_entries = [e for e in file_bom_entries if f"{e.get('archiveContext', '')}!{e['path']}" in reduced_path_set] version_report['detailedFileBomViewEntries'] = reduced_file_bom_entries component_identifiers = [f"{e['projectId']}:{e['versionId']}" for e in reduced_file_bom_entries] From 955b36b213cd58efe1d61334dc82d38109dc42f1 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 12 Jul 2024 09:26:39 -0400 Subject: [PATCH 090/146] remove double quotes inside f-interpolated string --- examples/client/file_hierarchy_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 74c1cdb4..4af4aafc 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -238,7 +238,7 @@ def main(): json.dump(version_report, f) # TODO items # Process file section of report data to identify primary paths - path_set = [f"{entry.get('archiveContext', "")}!{entry['path']}" for entry in version_report['detailedFileBomViewEntries']] + path_set = [f"{entry.get('archiveContext', '')}!{entry['path']}" for entry in version_report['detailedFileBomViewEntries']] reduced_path_set = reduce(path_set.copy()) logging.info(f"{len(path_set)-len(reduced_path_set)} path entries were scrubbed from the dataset.") From 786c439b8dcb3ac262fc88e5a640aa0c43affc76 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 12 Jul 2024 09:52:01 -0400 Subject: [PATCH 091/146] removed unused function --- examples/client/file_hierarchy_report.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 4af4aafc..15271a37 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -79,25 +79,6 @@ def find_project_version_by_name(bd, project, version_name): assert len(versions) == 1, f"Project version {version_name} for project {project['name']} not found" return versions[0] -def get_bd_project_data(hub_client, project_name, version_name): - """ Get and return project ID, version ID. """ - project_id = "" - for project in hub_client.get_resource("projects"): - if project['name'] == project_name: - project_id = (project['_meta']['href']).split("projects/", 1)[1] - break - if project_id == "": - sys.exit(f"No project for {project_name} was found!") - version_id = codelocations = "" - for version in hub_client.get_resource("versions", project): - if version['versionName'] == version_name: - version_id = (version['_meta']['href']).split("versions/", 1)[1] - break - if version_id == "": - sys.exit(f"No project version for {version_name} was found!") - - return project_id, version_id - def create_version_details_report(bd, version): version_reports_url = bd.list_resources(version).get('versionReport') post_data = { From 9ce7e7213cb43dc615b77eb5450dcff72c148cff Mon Sep 17 00:00:00 2001 From: dnichol Date: Tue, 25 Jun 2024 14:53:53 +0100 Subject: [PATCH 092/146] Hierarchy source report - basic structure and report generation --- examples/client/file_hierarchy_report.py | 265 +++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 examples/client/file_hierarchy_report.py diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py new file mode 100644 index 00000000..f756f3f6 --- /dev/null +++ b/examples/client/file_hierarchy_report.py @@ -0,0 +1,265 @@ +''' +Created on June 25, 2024 + +@author: dnichol and kumykov + +Generate version detail reports (source and components) and consolidate information on source matches, with license +and component matched. Removes matches found underneith other matched components in the source tree (configurable). + +Copyright (C) 2023 Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +''' + +import argparse +import logging +import sys +import os +import re +import time +import subprocess +import json +import traceback +import copy +import ijson +from blackduck import Client +from zipfile import ZipFile + +program_description = \ +'''Generate version detail reports (source and components) and consolidate information on source matches, with license +and component matched. Removes matches found underneith other matched components in the source tree (configurable). + +This script assumes a project version exists and has scans associated with it (i.e. the project is not scanned as part of this process). + +Config file: +API Token and Black Duck URL need to be placed in the .restconfig.json file which must be placed in the same folder where this script resides. + { + "baseurl": "https://hub-hostname", + "api_token": "", + "insecure": true or false , + "debug": true or false + } + +Remarks: +This script uses 3rd party PyPI package "ijson". This package must be installed. +''' + +# BD report general +BLACKDUCK_REPORT_MEDIATYPE = "application/vnd.blackducksoftware.report-4+json" +blackduck_report_download_api = "/api/projects/{projectId}/versions/{projectVersionId}/reports/{reportId}/download" +# BD version details report +blackduck_create_version_report_api = "/api/versions/{projectVersionId}/reports" +blackduck_version_report_filename = "./blackduck_version_report_for_{projectVersionId}.zip" +# Consolidated report +BLACKDUCK_VERSION_MEDIATYPE = "application/vnd.blackducksoftware.status-4+json" +BLACKDUCK_VERSION_API = "/api/current-version" +REPORT_DIR = "./blackduck_component_source_report" +# Retries to wait for BD report creation. RETRY_LIMIT can be overwritten by the script parameter. +RETRY_LIMIT = 30 +RETRY_TIMER = 30 + +def log_config(debug): + if debug: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.DEBUG) + else: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.INFO) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("blackduck").setLevel(logging.WARNING) + +def parse_parameter(): + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("project", + metavar="project", + type=str, + help="Provide the BlackDuck project name.") + parser.add_argument("version", + metavar="version", + type=str, + help="Provide the BlackDuck project version name.") + parser.add_argument("-kh", + "--keep_hierarchy", + action='store_true', + help="Set to keep all entries in the sources report. Will not remove components found under others.") + parser.add_argument("-rr", + "--report_retries", + metavar="", + type=int, + default=RETRY_LIMIT, + help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") + parser.add_argument("-t", + "--timeout", + metavar="", + type=int, + default=15, + help="Timeout for REST-API. Some API may take longer than the default 15 seconds") + parser.add_argument("-r", + "--retries", + metavar="", + type=int, + default=3, + help="Retries for REST-API. Some API may need more retries than the default 3 times") + return parser.parse_args() + +def get_bd_project_data(hub_client, project_name, version_name): + """ Get and return project ID, version ID. """ + project_id = "" + for project in hub_client.get_resource("projects"): + if project['name'] == project_name: + project_id = (project['_meta']['href']).split("projects/", 1)[1] + break + if project_id == "": + sys.exit(f"No project for {project_name} was found!") + version_id = codelocations = "" + for version in hub_client.get_resource("versions", project): + if version['versionName'] == version_name: + version_id = (version['_meta']['href']).split("versions/", 1)[1] + break + if version_id == "": + sys.exit(f"No project version for {version_name} was found!") + + return project_id, version_id + +def report_create(hub_client, url, body): + """ + Request BlackDuck to create report. Requested report is included in the request payload. + """ + res = hub_client.session.post(url, headers={'Content-Type': BLACKDUCK_REPORT_MEDIATYPE}, json=body) + if res.status_code != 201: + sys.exit(f"BlackDuck report creation failed with status {res.status_code}!") + return res.headers['Location'] # return report_url + +def report_download(hub_client, report_url, project_id, version_id, retries): + """ + Download the generated report after the report completion. We will retry until reaching the retry-limit. + """ + while retries: + res = hub_client.session.get(report_url, headers={'Accept': BLACKDUCK_REPORT_MEDIATYPE}) + if res.status_code == 200 and (json.loads(res.content))['status'] == "COMPLETED": + report_id = report_url.split("reports/", 1)[1] + download_url = (((blackduck_report_download_api.replace("{projectId}", project_id)) + .replace("{projectVersionId}", version_id)) + .replace("{reportId}", report_id)) + res = hub_client.session.get(download_url, + headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + if res.status_code != 200: + sys.exit(f"BlackDuck report download failed with status {res.status_code} for {download_url}!") + return res.content + elif res.status_code != 200: + sys.exit(f"BlackDuck report creation not completed successfully with status {res.status_code}") + else: + retries -= 1 + logging.info(f"Waiting for the report generation for {report_url} with the remaining retries {retries} times.") + time.sleep(RETRY_TIMER) + sys.exit(f"BlackDuck report for {report_url} was not generated after retries {RETRY_TIMER} sec * {retries} times!") + +def get_version_detail_report(hub_client, project_id, version_id, retries): + """ Create and get BOM component and BOM source file report in json. """ + create_version_url = blackduck_create_version_report_api.replace("{projectVersionId}", version_id) + body = { + 'reportFormat' : 'JSON', + 'locale' : 'en_US', + 'versionId' : f'{version_id}', + 'categories' : [ 'COMPONENTS', 'FILES' ] # Generating "project version" report including components and files + } + report_url = report_create(hub_client, create_version_url, body) + # Zipped report content is received and write the content to a local zip file + content = report_download(hub_client, report_url, project_id, version_id, retries) + output_file = blackduck_version_report_filename.replace("{projectVersionId}", version_id) + with open(output_file, "wb") as f: + f.write(content) + return output_file + +def get_blackduck_version(hub_client): + url = hub_client.base_url + BLACKDUCK_VERSION_API + res = hub_client.session.get(url) + if res.status_code == 200 and res.content: + return json.loads(res.content)['version'] + else: + sys.exit(f"Get BlackDuck version failed with status {res.status_code}") + +def generate_file_report(hub_client, project_id, version_id, keep_hierarchy, retries): + """ + Create a consolidated file report from BlackDuck project version source and components reports. + Remarks: + """ + if not os.path.exists(REPORT_DIR): + os.makedirs(REPORT_DIR) + + # Report body - Component BOM, file BOM with Discoveries data + version_report_zip = get_version_detail_report(hub_client, project_id, version_id, retries) + with ZipFile(f"./{version_report_zip}", "r") as vzf: + vzf.extractall() + for i, unzipped_version in enumerate(vzf.namelist()): + if re.search(r"\bversion.+json\b", unzipped_version) is not None: + break + if i + 1 >= len(vzf.namelist()): + sys.exit(f"Version detail file not found in the downloaded report: {version_report_zip}!") + + # Report body - Component BOM report + with open(f"./{unzipped_version}", "r") as uvf: + for i, comp_bom in enumerate(ijson.items(uvf, 'aggregateBomViewEntries.item')): + logging.info(f"{comp_bom['componentName']}") + logging.info(f"Number of the reported components {i+1}") + + +def main(): + args = parse_parameter() + debug = 0 + try: + if args.project == "": + sys.exit("Please set BlackDuck project name!") + if args.version == "": + sys.exit("Please set BlackDuck project version name!") + + with open(".restconfig.json", "r") as f: + config = json.load(f) + # Remove last slash if there is, otherwise REST API may fail. + if re.search(r".+/$", config['baseurl']): + bd_url = config['baseurl'][:-1] + else: + bd_url = config['baseurl'] + bd_token = config['api_token'] + bd_insecure = not config['insecure'] + if config['debug']: + debug = 1 + + log_config(debug) + + hub_client = Client(token=bd_token, + base_url=bd_url, + verify=bd_insecure, + timeout=args.timeout, + retries=args.retries) + + project_id, version_id = get_bd_project_data(hub_client, args.project, args.version) + + generate_file_report(hub_client, + project_id, + version_id, + args.keep_hierarchy, + args.report_retries + ) + + except (Exception, BaseException) as err: + logging.error(f"Exception by {str(err)}. See the stack trace") + traceback.print_exc() + +if __name__ == '__main__': + sys.exit(main()) From 7fd19bfda013fc5a1709df307c93a371d92ec013 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 9 Jul 2024 09:46:09 -0400 Subject: [PATCH 093/146] 0.5 prototype --- examples/client/file_hierarchy_report.py | 228 ++++++++++------------- 1 file changed, 94 insertions(+), 134 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index f756f3f6..0dba1657 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -30,6 +30,7 @@ import argparse import logging import sys +import io import os import re import time @@ -40,10 +41,11 @@ import ijson from blackduck import Client from zipfile import ZipFile +from pprint import pprint program_description = \ '''Generate version detail reports (source and components) and consolidate information on source matches, with license -and component matched. Removes matches found underneith other matched components in the source tree (configurable). +and component matched. Removes matches found underneath other matched components in the source tree (configurable). This script assumes a project version exists and has scans associated with it (i.e. the project is not scanned as part of this process). @@ -83,39 +85,21 @@ def log_config(debug): logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("blackduck").setLevel(logging.WARNING) -def parse_parameter(): - parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("project", - metavar="project", - type=str, - help="Provide the BlackDuck project name.") - parser.add_argument("version", - metavar="version", - type=str, - help="Provide the BlackDuck project version name.") - parser.add_argument("-kh", - "--keep_hierarchy", - action='store_true', - help="Set to keep all entries in the sources report. Will not remove components found under others.") - parser.add_argument("-rr", - "--report_retries", - metavar="", - type=int, - default=RETRY_LIMIT, - help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") - parser.add_argument("-t", - "--timeout", - metavar="", - type=int, - default=15, - help="Timeout for REST-API. Some API may take longer than the default 15 seconds") - parser.add_argument("-r", - "--retries", - metavar="", - type=int, - default=3, - help="Retries for REST-API. Some API may need more retries than the default 3 times") - return parser.parse_args() +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + assert len(projects) == 1, f"Project {project_name} not found." + return projects[0] + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + assert len(versions) == 1, f"Project version {version_name} for project {project['name']} not found" + return versions[0] def get_bd_project_data(hub_client, project_name, version_name): """ Get and return project ID, version ID. """ @@ -136,55 +120,53 @@ def get_bd_project_data(hub_client, project_name, version_name): return project_id, version_id -def report_create(hub_client, url, body): - """ - Request BlackDuck to create report. Requested report is included in the request payload. - """ - res = hub_client.session.post(url, headers={'Content-Type': BLACKDUCK_REPORT_MEDIATYPE}, json=body) - if res.status_code != 201: - sys.exit(f"BlackDuck report creation failed with status {res.status_code}!") - return res.headers['Location'] # return report_url - -def report_download(hub_client, report_url, project_id, version_id, retries): - """ - Download the generated report after the report completion. We will retry until reaching the retry-limit. - """ - while retries: - res = hub_client.session.get(report_url, headers={'Accept': BLACKDUCK_REPORT_MEDIATYPE}) - if res.status_code == 200 and (json.loads(res.content))['status'] == "COMPLETED": - report_id = report_url.split("reports/", 1)[1] - download_url = (((blackduck_report_download_api.replace("{projectId}", project_id)) - .replace("{projectVersionId}", version_id)) - .replace("{reportId}", report_id)) - res = hub_client.session.get(download_url, - headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) - if res.status_code != 200: - sys.exit(f"BlackDuck report download failed with status {res.status_code} for {download_url}!") - return res.content - elif res.status_code != 200: - sys.exit(f"BlackDuck report creation not completed successfully with status {res.status_code}") - else: - retries -= 1 - logging.info(f"Waiting for the report generation for {report_url} with the remaining retries {retries} times.") - time.sleep(RETRY_TIMER) - sys.exit(f"BlackDuck report for {report_url} was not generated after retries {RETRY_TIMER} sec * {retries} times!") - -def get_version_detail_report(hub_client, project_id, version_id, retries): - """ Create and get BOM component and BOM source file report in json. """ - create_version_url = blackduck_create_version_report_api.replace("{projectVersionId}", version_id) - body = { +def create_version_details_report(bd, version): + version_reports_url = bd.list_resources(version).get('versionReport') + post_data = { 'reportFormat' : 'JSON', 'locale' : 'en_US', - 'versionId' : f'{version_id}', + 'versionId': version['_meta']['href'].split("/")[-1], 'categories' : [ 'COMPONENTS', 'FILES' ] # Generating "project version" report including components and files } - report_url = report_create(hub_client, create_version_url, body) - # Zipped report content is received and write the content to a local zip file - content = report_download(hub_client, report_url, project_id, version_id, retries) - output_file = blackduck_version_report_filename.replace("{projectVersionId}", version_id) - with open(output_file, "wb") as f: - f.write(content) - return output_file + + bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json" + r = bd.session.post(version_reports_url, json=post_data) + if (r.status_code == 403): + logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") + r.raise_for_status() + pprint(r.headers) + location = r.headers.get('Location') + assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" + return location + +def download_report(bd, location, retries): + report_id = location.split("/")[-1] + print (location) + url_data = location.split('/') + url_data.pop(4) + url_data.pop(4) + download_link = '/'.join(url_data) + print(download_link) + if retries: + logging.debug(f"Retrieving generated report from {location}") + response = bd.session.get(location) + report_status = response.json().get('status', 'Not Ready') + if response.status_code == 200 and report_status == 'COMPLETED': + response = bd.session.get(download_link, headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + pprint(response) + if response.status_code == 200: + return response.content + else: + logging.error("Ruh-roh, not sure what happened here") + return None + else: + logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {retries} seconds then retrying...") + time.sleep(60) + retries -= 1 + return download_report(bd, location, retries) + else: + logging.debug(f"Failed to retrieve report {report_id} after multiple retries") + return None def get_blackduck_version(hub_client): url = hub_client.base_url + BLACKDUCK_VERSION_API @@ -194,68 +176,46 @@ def get_blackduck_version(hub_client): else: sys.exit(f"Get BlackDuck version failed with status {res.status_code}") -def generate_file_report(hub_client, project_id, version_id, keep_hierarchy, retries): - """ - Create a consolidated file report from BlackDuck project version source and components reports. - Remarks: - """ - if not os.path.exists(REPORT_DIR): - os.makedirs(REPORT_DIR) - - # Report body - Component BOM, file BOM with Discoveries data - version_report_zip = get_version_detail_report(hub_client, project_id, version_id, retries) - with ZipFile(f"./{version_report_zip}", "r") as vzf: - vzf.extractall() - for i, unzipped_version in enumerate(vzf.namelist()): - if re.search(r"\bversion.+json\b", unzipped_version) is not None: - break - if i + 1 >= len(vzf.namelist()): - sys.exit(f"Version detail file not found in the downloaded report: {version_report_zip}!") - - # Report body - Component BOM report - with open(f"./{unzipped_version}", "r") as uvf: - for i, comp_bom in enumerate(ijson.items(uvf, 'aggregateBomViewEntries.item')): - logging.info(f"{comp_bom['componentName']}") - logging.info(f"Number of the reported components {i+1}") - +def parse_command_args(): + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") + parser.add_argument("-pn", "--project-name", required=True, help="Project Name") + parser.add_argument("-pv", "--project-version-name", required=True, help="Project Version Name") + parser.add_argument("-kh", "--keep_hierarchy", action='store_true', help="Set to keep all entries in the sources report. Will not remove components found under others.") + parser.add_argument("--report-retries", metavar="", type=int, default=RETRY_LIMIT, help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") + parser.add_argument("--timeout", metavar="", type=int, default=60, help="Timeout for REST-API. Some API may take longer than the default 60 seconds") + parser.add_argument("--retries", metavar="", type=int, default=4, help="Retries for REST-API. Some API may need more retries than the default 4 times") + return parser.parse_args() def main(): - args = parse_parameter() - debug = 0 + args = parse_command_args() + with open(args.token_file, 'r') as tf: + token = tf.readline().strip() try: - if args.project == "": - sys.exit("Please set BlackDuck project name!") - if args.version == "": - sys.exit("Please set BlackDuck project version name!") - - with open(".restconfig.json", "r") as f: - config = json.load(f) - # Remove last slash if there is, otherwise REST API may fail. - if re.search(r".+/$", config['baseurl']): - bd_url = config['baseurl'][:-1] - else: - bd_url = config['baseurl'] - bd_token = config['api_token'] - bd_insecure = not config['insecure'] - if config['debug']: - debug = 1 - - log_config(debug) - - hub_client = Client(token=bd_token, - base_url=bd_url, - verify=bd_insecure, + log_config(args.debug) + hub_client = Client(token=token, + base_url=args.base_url, + verify=args.no_verify, timeout=args.timeout, retries=args.retries) - project_id, version_id = get_bd_project_data(hub_client, args.project, args.version) + project = find_project_by_name(hub_client, args.project_name) + version = find_project_version_by_name(hub_client, project, args.project_version_name) + pprint(version) + location = create_version_details_report(hub_client, version) + pprint(location) + report_zip = download_report(hub_client, location, args.report_retries) + pprint(report_zip) + logging.debug(f"Deleting report from Black Duck {hub_client.session.delete(location)}") + zip=ZipFile(io.BytesIO(report_zip), "r") + pprint(zip.namelist()) + report_data = {name: zip.read(name) for name in zip.namelist()} + filename = [i for i in report_data.keys() if i.endswith(".json")][0] + pprint(json.loads(report_data[filename])) - generate_file_report(hub_client, - project_id, - version_id, - args.keep_hierarchy, - args.report_retries - ) except (Exception, BaseException) as err: logging.error(f"Exception by {str(err)}. See the stack trace") From 3297d551c02f9dc2b8ea62fc802dc2281703c70a Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 9 Jul 2024 10:20:35 -0400 Subject: [PATCH 094/146] 0.5 prototype --- examples/client/file_hierarchy_report.py | 33 ++++++------------------ 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 0dba1657..d380dd9f 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -31,14 +31,9 @@ import logging import sys import io -import os -import re import time -import subprocess import json import traceback -import copy -import ijson from blackduck import Client from zipfile import ZipFile from pprint import pprint @@ -49,17 +44,6 @@ This script assumes a project version exists and has scans associated with it (i.e. the project is not scanned as part of this process). -Config file: -API Token and Black Duck URL need to be placed in the .restconfig.json file which must be placed in the same folder where this script resides. - { - "baseurl": "https://hub-hostname", - "api_token": "", - "insecure": true or false , - "debug": true or false - } - -Remarks: -This script uses 3rd party PyPI package "ijson". This package must be installed. ''' # BD report general @@ -134,26 +118,24 @@ def create_version_details_report(bd, version): if (r.status_code == 403): logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") r.raise_for_status() - pprint(r.headers) location = r.headers.get('Location') assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" return location def download_report(bd, location, retries): report_id = location.split("/")[-1] - print (location) + logging.debug(f"Report location {location}") url_data = location.split('/') url_data.pop(4) url_data.pop(4) download_link = '/'.join(url_data) - print(download_link) + logging.debug(f"Report Download link {download_link}") if retries: - logging.debug(f"Retrieving generated report from {location}") + logging.debug(f"Retrieving generated report for {location} via {download_link}") response = bd.session.get(location) report_status = response.json().get('status', 'Not Ready') if response.status_code == 200 and report_status == 'COMPLETED': response = bd.session.get(download_link, headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) - pprint(response) if response.status_code == 200: return response.content else: @@ -204,17 +186,18 @@ def main(): project = find_project_by_name(hub_client, args.project_name) version = find_project_version_by_name(hub_client, project, args.project_version_name) - pprint(version) location = create_version_details_report(hub_client, version) - pprint(location) report_zip = download_report(hub_client, location, args.report_retries) - pprint(report_zip) logging.debug(f"Deleting report from Black Duck {hub_client.session.delete(location)}") zip=ZipFile(io.BytesIO(report_zip), "r") pprint(zip.namelist()) report_data = {name: zip.read(name) for name in zip.namelist()} filename = [i for i in report_data.keys() if i.endswith(".json")][0] - pprint(json.loads(report_data[filename])) + version_report = json.loads(report_data[filename]) + # TODO items + # Process file section of report data to identify primary paths + # Combine component data with selected file data + # Output result with CSV anf JSON as options. except (Exception, BaseException) as err: From 63d975ac91ef5d2251d94ac2c1a865840495445e Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 10 Jul 2024 13:34:55 -0400 Subject: [PATCH 095/146] . --- examples/client/file_hierarchy_report.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index d380dd9f..c86d06da 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -194,6 +194,8 @@ def main(): report_data = {name: zip.read(name) for name in zip.namelist()} filename = [i for i in report_data.keys() if i.endswith(".json")][0] version_report = json.loads(report_data[filename]) + with open("out.json", "w") as f: + json.dump(version_report, f) # TODO items # Process file section of report data to identify primary paths # Combine component data with selected file data From 008cc55808d5860ab44d5b60a9177b8fb0083025 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 11 Jul 2024 15:57:04 -0400 Subject: [PATCH 096/146] Functional requirement satisfied --- examples/client/file_hierarchy_report.py | 78 ++++++++++++++++++++---- 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index c86d06da..f97e96b4 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -28,6 +28,7 @@ ''' import argparse +import csv import logging import sys import io @@ -47,15 +48,8 @@ ''' # BD report general -BLACKDUCK_REPORT_MEDIATYPE = "application/vnd.blackducksoftware.report-4+json" -blackduck_report_download_api = "/api/projects/{projectId}/versions/{projectVersionId}/reports/{reportId}/download" -# BD version details report -blackduck_create_version_report_api = "/api/versions/{projectVersionId}/reports" -blackduck_version_report_filename = "./blackduck_version_report_for_{projectVersionId}.zip" -# Consolidated report BLACKDUCK_VERSION_MEDIATYPE = "application/vnd.blackducksoftware.status-4+json" BLACKDUCK_VERSION_API = "/api/current-version" -REPORT_DIR = "./blackduck_component_source_report" # Retries to wait for BD report creation. RETRY_LIMIT can be overwritten by the script parameter. RETRY_LIMIT = 30 RETRY_TIMER = 30 @@ -122,7 +116,7 @@ def create_version_details_report(bd, version): assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" return location -def download_report(bd, location, retries): +def download_report(bd, location, retries, timeout): report_id = location.split("/")[-1] logging.debug(f"Report location {location}") url_data = location.split('/') @@ -142,10 +136,10 @@ def download_report(bd, location, retries): logging.error("Ruh-roh, not sure what happened here") return None else: - logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {retries} seconds then retrying...") - time.sleep(60) + logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {timeout} seconds then retrying...") + time.sleep(timeout) retries -= 1 - return download_report(bd, location, retries) + return download_report(bd, location, retries, timeout) else: logging.debug(f"Failed to retrieve report {report_id} after multiple retries") return None @@ -158,6 +152,47 @@ def get_blackduck_version(hub_client): else: sys.exit(f"Get BlackDuck version failed with status {res.status_code}") +def reduce(path_set): + path_set.sort() + for path in path_set: + if len(path) < 3: + continue + index = path_set.index(path) + while index + 1 < len(path_set) and path in path_set[index+1]: + logging.debug(f"{path} is in {path_set[index+1]} deleting the sub-path from the list") + path_set.pop(index+1) + return path_set + +def trim_version_report(version_report, reduced_path_set): + file_bom_entries = version_report['detailedFileBomViewEntries'] + aggregate_bom_view_entries = version_report['aggregateBomViewEntries'] + + reduced_file_bom_entries = [e for e in file_bom_entries if f"{e.get('archiveContext', "")}!{e['path']}" in reduced_path_set] + version_report['detailedFileBomViewEntries'] = reduced_file_bom_entries + + component_identifiers = [f"{e['projectId']}:{e['versionId']}" for e in reduced_file_bom_entries] + deduplicated = list(dict.fromkeys(component_identifiers)) + + reduced_aggregate_bom_view_entries = [e for e in aggregate_bom_view_entries if f"{e['producerProject']['id']}:{e['producerReleases'][0]['id']}" in deduplicated] + version_report['aggregateBomViewEntries'] = reduced_aggregate_bom_view_entries + +def write_output_file(version_report, output_file): + if output_file.lower().endswith(".csv"): + logging.info(f"Writing CSV output into {output_file}") + field_names = list(version_report['aggregateBomViewEntries'][0].keys()) + with open(output_file, "w") as f: + writer = csv.DictWriter(f, fieldnames = field_names) + writer.writeheader() + writer.writerows(version_report['aggregateBomViewEntries']) + + return + # If it's neither, then .json + if not output_file.lower().endswith(".json"): + output_file += ".json" + logging.info(f"Writing JSON output into {output_file}") + with open(output_file,"w") as f: + json.dump(version_report, f) + def parse_command_args(): parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") @@ -166,8 +201,10 @@ def parse_command_args(): parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") parser.add_argument("-pn", "--project-name", required=True, help="Project Name") parser.add_argument("-pv", "--project-version-name", required=True, help="Project Version Name") + parser.add_argument("-o", "--output-file", required=False, help="File name to write output. File extension determines format .json and .csv, json is the default.") parser.add_argument("-kh", "--keep_hierarchy", action='store_true', help="Set to keep all entries in the sources report. Will not remove components found under others.") parser.add_argument("--report-retries", metavar="", type=int, default=RETRY_LIMIT, help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") + parser.add_argument("--report-timeout", metavar="", type=int, default=RETRY_TIMER, help="Wait time between subsequent download attempts.") parser.add_argument("--timeout", metavar="", type=int, default=60, help="Timeout for REST-API. Some API may take longer than the default 60 seconds") parser.add_argument("--retries", metavar="", type=int, default=4, help="Retries for REST-API. Some API may need more retries than the default 4 times") return parser.parse_args() @@ -176,6 +213,9 @@ def main(): args = parse_command_args() with open(args.token_file, 'r') as tf: token = tf.readline().strip() + output_file = args.output_file + if not args.output_file: + output_file = f"{args.project_name}-{args.project_version_name}.json".replace(" ","_") try: log_config(args.debug) hub_client = Client(token=token, @@ -187,7 +227,7 @@ def main(): project = find_project_by_name(hub_client, args.project_name) version = find_project_version_by_name(hub_client, project, args.project_version_name) location = create_version_details_report(hub_client, version) - report_zip = download_report(hub_client, location, args.report_retries) + report_zip = download_report(hub_client, location, args.report_retries, args.report_timeout) logging.debug(f"Deleting report from Black Duck {hub_client.session.delete(location)}") zip=ZipFile(io.BytesIO(report_zip), "r") pprint(zip.namelist()) @@ -198,10 +238,24 @@ def main(): json.dump(version_report, f) # TODO items # Process file section of report data to identify primary paths + path_set = [f"{entry.get('archiveContext', "")}!{entry['path']}" for entry in version_report['detailedFileBomViewEntries']] + reduced_path_set = reduce(path_set.copy()) + logging.info(f"{len(path_set)-len(reduced_path_set)} path entries were scrubbed from the dataset.") + + # Remove component entries that correspond to removed path entries. + + logging.info(f"Original dataset contains {len(version_report['aggregateBomViewEntries'])} bom entries and {len(version_report['detailedFileBomViewEntries'])} file view entries") + if not args.keep_hierarchy: + trim_version_report(version_report, reduced_path_set) + logging.info(f"Truncated dataset contains {len(version_report['aggregateBomViewEntries'])} bom entries and {len(version_report['detailedFileBomViewEntries'])} file view entries") + + write_output_file(version_report, output_file) + # Combine component data with selected file data # Output result with CSV anf JSON as options. + except (Exception, BaseException) as err: logging.error(f"Exception by {str(err)}. See the stack trace") traceback.print_exc() From 3fab2a245c1a05dcd6b36ec2102fa858b7f748a6 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 12 Jul 2024 09:23:08 -0400 Subject: [PATCH 097/146] remove double quotes inside f-interpolated string --- examples/client/file_hierarchy_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index f97e96b4..74c1cdb4 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -167,7 +167,7 @@ def trim_version_report(version_report, reduced_path_set): file_bom_entries = version_report['detailedFileBomViewEntries'] aggregate_bom_view_entries = version_report['aggregateBomViewEntries'] - reduced_file_bom_entries = [e for e in file_bom_entries if f"{e.get('archiveContext', "")}!{e['path']}" in reduced_path_set] + reduced_file_bom_entries = [e for e in file_bom_entries if f"{e.get('archiveContext', '')}!{e['path']}" in reduced_path_set] version_report['detailedFileBomViewEntries'] = reduced_file_bom_entries component_identifiers = [f"{e['projectId']}:{e['versionId']}" for e in reduced_file_bom_entries] From 8f15806ca52530d9c0139962377e762589e7cd89 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 12 Jul 2024 09:26:39 -0400 Subject: [PATCH 098/146] remove double quotes inside f-interpolated string --- examples/client/file_hierarchy_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 74c1cdb4..4af4aafc 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -238,7 +238,7 @@ def main(): json.dump(version_report, f) # TODO items # Process file section of report data to identify primary paths - path_set = [f"{entry.get('archiveContext', "")}!{entry['path']}" for entry in version_report['detailedFileBomViewEntries']] + path_set = [f"{entry.get('archiveContext', '')}!{entry['path']}" for entry in version_report['detailedFileBomViewEntries']] reduced_path_set = reduce(path_set.copy()) logging.info(f"{len(path_set)-len(reduced_path_set)} path entries were scrubbed from the dataset.") From 3e2bf60c6cc065be09e959f41de9fa8b037f1165 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 15 Jul 2024 09:51:08 -0400 Subject: [PATCH 099/146] extrasaction on csv export --- examples/client/file_hierarchy_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 15271a37..da9e5db0 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -162,7 +162,7 @@ def write_output_file(version_report, output_file): logging.info(f"Writing CSV output into {output_file}") field_names = list(version_report['aggregateBomViewEntries'][0].keys()) with open(output_file, "w") as f: - writer = csv.DictWriter(f, fieldnames = field_names) + writer = csv.DictWriter(f, fieldnames = field_names, extrasaction = 'ignore') writer.writeheader() writer.writerows(version_report['aggregateBomViewEntries']) From afe2a8e7d24b4cf7e0f6aa40fa827df074223b9d Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 15 Jul 2024 09:59:21 -0400 Subject: [PATCH 100/146] . --- examples/client/file_hierarchy_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index da9e5db0..616fd836 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -162,7 +162,7 @@ def write_output_file(version_report, output_file): logging.info(f"Writing CSV output into {output_file}") field_names = list(version_report['aggregateBomViewEntries'][0].keys()) with open(output_file, "w") as f: - writer = csv.DictWriter(f, fieldnames = field_names, extrasaction = 'ignore') + writer = csv.DictWriter(f, fieldnames = field_names, extrasaction = 'ignore') # TODO writer.writeheader() writer.writerows(version_report['aggregateBomViewEntries']) From 4b7358fed69c536ac51ee9de55aed7754168e706 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 15 Jul 2024 10:46:48 -0400 Subject: [PATCH 101/146] . --- .../multi-image/scan_docker_image_lite.py | 47 +++++++------------ 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index 65703190..e688720e 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -4,7 +4,7 @@ Alternative version if Docker image layer by layer scan. -This program will download docker image and scan it into Blackduck server layer by layer +This program will download docker image and scan it into Black Duck server layer by layer Each layer will be scanned as a separate scan with a signature scan. Layers in the container images could be grouped into groups of contiguous layers. @@ -49,16 +49,16 @@ optional arguments: -h, --help show this help message and exit - --grouping GROUPING Group layers into user defined provect versions (can't be used with --base-image) + --grouping GROUPING Group layers into user defined project versions (can't be used with --base-image) --base-image BASE_IMAGE Use base image spec to determine base image/layers (can't be used with --grouping or --dockerfile) --dockerfile DOCKERFILE Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image) --project-name Specify project name (default is container image spec) - --project-verson Specify project version (default is container image tag/version) + --project-version Specify project version (default is container image tag/version) --detect-options DETECT_OPTIONS - Extra detect options to be passed directlyto the detect + Extra detect options to be passed directly to the detect Using --detect-options @@ -187,13 +187,23 @@ def read_config(self): with open(configFile) as fp: data = json.load(fp) return data + + def read_oci_layout(self): + oci_layout_file = self.imagedir + "/" + 'oci-layout' + if os.path.exists(oci_layout_file) and os.path.isfile(oci_layout_file): + with open(oci_layout_file) as fp: + data = json.load(fp) + return data + else: + return None class Detector(): def __init__(self, hub): # self.detecturl = 'https://blackducksoftware.github.io/hub-detect/hub-detect.sh' # self.detecturl = 'https://detect.synopsys.com/detect.sh' # self.detecturl = 'https://detect.synopsys.com/detect7.sh' - self.detecturl = 'https://detect.synopsys.com/detect8.sh' + # self.detecturl = 'https://detect.synopsys.com/detect8.sh' + self.detecturl = 'https://detect.synopsys.com/detect9.sh' self.baseurl = hub.config['baseurl'] self.filename = '/tmp/hub-detect.sh' self.token=hub.config['api_token'] @@ -240,6 +250,7 @@ def __init__( if detect_options: self.extra_options = detect_options.split(" ") print ("<--{}-->".format(self.grouping)) + self.binary = False def prepare_container_image(self): self.docker.initdir() @@ -262,28 +273,6 @@ def prepare_container_image(self): if len(history_grouping) and self.grouping == '1024:everything': self.grouping = history_grouping - def prepare_container_image_old(self): - self.docker.initdir() - self.docker.pull_container_image(self.container_image_name) - result = self.docker.get_container_image_history(self.container_image_name) - history = result.stdout.splitlines() - layer_count = 0 - history_grouping = '' - for line in reversed(history): - print (line) - if not line.rstrip().endswith(b' 0B'): - layer_count +=1 - match = re.search('echo (.+?)_group_end', str(line)) - if match: - found = match.group(1) - if len(history_grouping): - history_grouping += ',' - history_grouping += str(layer_count) + ":" + found - if len(history_grouping) and self.grouping == '1024:everything': - self.grouping = history_grouping - self.docker.save_container_image(self.container_image_name) - self.docker.unravel_container() - def process_container_image_by_user_defined_groups(self): self.manifest = self.docker.read_manifest() print(self.manifest) @@ -455,12 +444,12 @@ def main(argv=None): parser = ArgumentParser() parser.add_argument('imagespec', help="Container image tag, e.g. repository/imagename:version") - parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined provect versions (can't be used with --base-image)") + parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined project versions (can't be used with --base-image)") parser.add_argument('--base-image',default=None, type=str, help="Use base image spec to determine base image/layers (can't be used with --grouping or --dockerfile)") parser.add_argument('--dockerfile',default=None, type=str, help="Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image)") parser.add_argument('--project-name',default=None, type=str, help="Specify project name (default is container image spec)") parser.add_argument('--project-version',default=None, type=str, help="Specify project version (default is container image tag/version)") - parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directlyto the detect") + parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directly to the detect") args = parser.parse_args() From 918e6958f928b4808469ef7557d73bdae885412e Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 15 Jul 2024 14:36:42 -0400 Subject: [PATCH 102/146] updated oci image support --- examples/client/multi-image/generate-clone.sh | 4 +- .../multi-image/manage_project_structure.py | 5 +- .../multi-image/scan_docker_image_lite.py | 125 +++++++++++++----- 3 files changed, 101 insertions(+), 33 deletions(-) diff --git a/examples/client/multi-image/generate-clone.sh b/examples/client/multi-image/generate-clone.sh index ca90b853..c968f0d6 100644 --- a/examples/client/multi-image/generate-clone.sh +++ b/examples/client/multi-image/generate-clone.sh @@ -15,6 +15,6 @@ ls -l $SPECFILE COMMAND="python3 examples/client/multi-image/manage_project_structure.py" -$COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ -# $COMMAND -u $BD_URL -t token -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@ +# $COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ +$COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@ diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 0aa48dff..99ca89dd 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -360,9 +360,12 @@ def scan_container_images(scan_params, hub): params['project'], params['version'], detect_options, - hub=hub + hub=hub, + binary=False ) except Exception: + import traceback + traceback.print_exc() logging.error(f"Scanning of {params['image']} failed, skipping") skipped_scans.append(params) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index e688720e..d43875c8 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -94,6 +94,7 @@ from pprint import pprint from sys import argv import json +import logging import os import requests import shutil @@ -136,7 +137,6 @@ def locate_docker(self): proc = subprocess.Popen(['which','docker'], stdout=subprocess.PIPE) out, err = proc.communicate() lines = out.decode().split('\n') - print(lines) if 'docker' in lines[0]: return lines[0] else: @@ -147,7 +147,7 @@ def pull_container_image(self, image_name): args.append(self.docker_path) args.append('pull') args.append(image_name) - return subprocess.run(args) + return subprocess.run(args, capture_output=True) def get_container_image_history(self, image_name): args = [] @@ -173,7 +173,7 @@ def unravel_container(self): args.append(self.imagefile) args.append('-C') args.append(self.imagedir) - return subprocess.run(args) + return subprocess.run(args, capture_output=True) def read_manifest(self): filename = self.imagedir + "/manifest.json" @@ -222,7 +222,7 @@ def detect_run(self, options=['--help']): cmd.append('--blackduck.api.token=' + self.token) cmd.append('--blackduck.trust.cert=true') cmd.extend(options) - subprocess.run(cmd) + return subprocess.run(cmd, capture_output=True) class ContainerImageScanner(): @@ -249,7 +249,6 @@ def __init__( self.extra_options = [] if detect_options: self.extra_options = detect_options.split(" ") - print ("<--{}-->".format(self.grouping)) self.binary = False def prepare_container_image(self): @@ -272,12 +271,11 @@ def prepare_container_image(self): history_grouping += str(layer_count) + ":" + found if len(history_grouping) and self.grouping == '1024:everything': self.grouping = history_grouping + self.oci_layout = self.docker.read_oci_layout() def process_container_image_by_user_defined_groups(self): self.manifest = self.docker.read_manifest() - print(self.manifest) self.config = self.docker.read_config() - print (json.dumps(self.config, indent=4)) if self.grouping: self.groups = dict(x.split(":") for x in self.grouping.split(",")) @@ -309,14 +307,12 @@ def process_container_image_by_user_defined_groups(self): layer['shaid'] = self.config['rootfs']['diff_ids'][num - 1] self.layers.append(layer) num = num + 1 - print (json.dumps(self.layers, indent=4)) + # print (json.dumps(self.layers, indent=4)) def process_container_image_by_base_image_info(self): self.manifest = self.docker.read_manifest() - print(self.manifest) self.config = self.docker.read_config() - print (json.dumps(self.config, indent=4)) - + self.layers = [] num = 1 offset = 0 @@ -342,27 +338,92 @@ def process_container_image_by_base_image_info(self): layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(num) self.layers.append(layer) num = num + 1 - print (json.dumps(self.layers, indent=4)) + # print (json.dumps(self.layers, indent=4)) + + def process_oci_container_image_by_user_defined_groups(self): + self.manifest = self.docker.read_manifest() + self.config = self.docker.read_config() + + self.layers = self.config['history'] + tagged_layers = [x for x in self.layers if '_group_end' in x.get('created_by')] + groups = {re.search('echo (.+?)_group_end', str(x['created_by'])).group(1): self.layers.index(x) for x in tagged_layers} + logging.debug(f"Container configuration defines following groups {groups}") + layer_paths = self.manifest[0]['Layers'].copy() + empty_layers = [x for x in self.layers if x.get('empty_layer', False)] + logging.debug(f"Total layers: {len(self.layers)} total paths: {len(layer_paths)} empty layers: {len(empty_layers)}") + + assert len(self.layers) == len(layer_paths) + len(empty_layers), "Something is wrong with this image, Layer math does not add up." + + for layer in self.layers: + layer['index'] = self.layers.index(layer) + if self.grouping: + layer['group_name'] = self.get_group_name(groups, layer['index']) + layer['project_name'] = "{}_{}".format(self.project_name,layer['group_name']) + layer['project_version'] = self.project_version + layer['name'] = "{}_{}_{}_layer_{}".format(self.project_name,self.project_version,layer['group_name'],str(layer['index'])) + else: + layer['project_name'] = self.project_name + layer['project_version'] = self.project_version + layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(layer['index']) + if not layer.get('empty_layer', False): + layer['path'] = layer_paths.pop(0) + # print (json.dumps(self.layers, indent=4)) + + def get_group_name(self, groups, index): + group_name = 'undefined' + for group, value in groups.items(): + if index <= value: + group_name = group + break + return group_name + + def process_oci_container_image_by_base_image_info(self): + print ("Processing by BAse Image not supported for OCI images") + sys.exit(1) + pass def process_container_image(self): + if self.oci_layout: + self.process_oci_container_image() + else: + self.process_docker_container_image() + + def process_docker_container_image(self): if self.grouping: self.process_container_image_by_user_defined_groups() else: self.process_container_image_by_base_image_info() + def process_oci_container_image(self): + if self.grouping: + self.process_oci_container_image_by_user_defined_groups() + else: + self.process_oci_container_image_by_base_image_info() + def submit_layer_scans(self): for layer in self.layers: - options = [] - options.append('--detect.project.name={}'.format(layer['project_name'])) - options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) - # options.append('--detect.blackduck.signature.scanner.disabled=false') - options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) - options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) - if self.base_image or self.grouping or self.dockerfile: - options.extend(self.adorn_extra_options(layer)) - else: - options.extend(self.extra_options) - self.hub_detect.detect_run(options) + if not layer.get('empty_layer', False): + options = [] + options.append('--detect.project.name={}'.format(layer['project_name'])) + options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) + options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) + if self.binary: + options.append('--detect.tools=BINARY_SCAN') + options.append('--detect.binary.scan.file.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.tools=SIGNATURE_SCAN') + if self.oci_layout: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) + if self.base_image or self.grouping or self.dockerfile: + options.extend(self.adorn_extra_options(layer)) + else: + options.extend(self.extra_options) + logging.info(f"Submitting scan for {layer['name']}") + completed = self.hub_detect.detect_run(options) + logging.info(f"Detect run for {layer['name']} completed with returncode {completed.returncode}") + def adorn_extra_options(self, layer): result = list() @@ -396,7 +457,7 @@ def get_base_layers(self): if self.base_image: imagelist.append(self.base_image) - print (imagelist) + # print (imagelist) base_layers = [] for image in imagelist: self.docker.initdir() @@ -404,16 +465,16 @@ def get_base_layers(self): self.docker.save_container_image(image) self.docker.unravel_container() manifest = self.docker.read_manifest() - print(manifest) + # print(manifest) config = self.docker.read_config() - print(config) + # print(config) base_layers.extend(config['rootfs']['diff_ids']) return base_layers def scan_container_image( imagespec, grouping=None, base_image=None, dockerfile=None, - project_name=None, project_version=None, detect_options=None, hub=None): + project_name=None, project_version=None, detect_options=None, hub=None, binary=False): if hub: hub = hub @@ -431,6 +492,8 @@ def scan_container_image( scanner.grouping = '1024:everything' else: scanner.base_layers = scanner.get_base_layers() + if binary: + scanner.binary = True scanner.prepare_container_image() scanner.process_container_image() scanner.submit_layer_scans() @@ -450,10 +513,11 @@ def main(argv=None): parser.add_argument('--project-name',default=None, type=str, help="Specify project name (default is container image spec)") parser.add_argument('--project-version',default=None, type=str, help="Specify project version (default is container image tag/version)") parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directly to the detect") - + parser.add_argument('--binary', action='store_true', help="Use Binary Scan instead of signature scan") + args = parser.parse_args() - print (args); + logging.debug(args); if not args.imagespec: parser.print_help(sys.stdout) @@ -474,7 +538,8 @@ def main(argv=None): args.dockerfile, args.project_name, args.project_version, - args.detect_options) + args.detect_options, + args.binary) if __name__ == "__main__": From 155445ff9fab04ad2520ddb3c2bdb91e17f8b9e9 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 16 Jul 2024 10:02:56 -0400 Subject: [PATCH 103/146] CSV format updated --- examples/client/file_hierarchy_report.py | 36 ++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 616fd836..d7e8992d 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -157,15 +157,45 @@ def trim_version_report(version_report, reduced_path_set): reduced_aggregate_bom_view_entries = [e for e in aggregate_bom_view_entries if f"{e['producerProject']['id']}:{e['producerReleases'][0]['id']}" in deduplicated] version_report['aggregateBomViewEntries'] = reduced_aggregate_bom_view_entries +''' + +CSV output details + +component name = aggregateBomViewEntries[].producerProject.name +version name = aggregateBomViewEntries[].producerReleases[0].version +license = licenses[].licenseDisplay +file path = extract from detailedFileBomViewEntries +match type = aggregateBomViewEntries[].matchTypes +review status = aggregateBomViewEntries[].reviewSummary.reviewStatus + +''' +def get_csv_fieldnames(): + return ['component name', 'version name', 'license', 'file path', 'match type', 'review status'] + +def get_csv_data(version_report): + csv_data = list() + for bom_view_entry in version_report['aggregateBomViewEntries']: + entry = dict() + entry['component name'] = bom_view_entry['producerProject']['name'] + entry['version name'] = bom_view_entry['producerReleases'][0]['version'] + entry['license'] = bom_view_entry['licenses'][0]['licenseDisplay'].replace(' AND ',';').replace('(','').replace(')','') + pid = bom_view_entry['producerProject']['id'] + vid = bom_view_entry['producerReleases'][0]['id'] + path_list = [p['path'] for p in version_report['detailedFileBomViewEntries'] if p['projectId'] == pid and p['versionId'] == vid] + entry['file path'] = ';'.join(path_list) + entry['match type'] = ';'.join(bom_view_entry['matchTypes']) + entry['review status'] = bom_view_entry['reviewSummary']['reviewStatus'] + csv_data.append(entry) + return csv_data + def write_output_file(version_report, output_file): if output_file.lower().endswith(".csv"): logging.info(f"Writing CSV output into {output_file}") - field_names = list(version_report['aggregateBomViewEntries'][0].keys()) + field_names = get_csv_fieldnames() with open(output_file, "w") as f: writer = csv.DictWriter(f, fieldnames = field_names, extrasaction = 'ignore') # TODO writer.writeheader() - writer.writerows(version_report['aggregateBomViewEntries']) - + writer.writerows(get_csv_data(version_report)) return # If it's neither, then .json if not output_file.lower().endswith(".json"): From f2fbad0d3dd73531f5b9383d6208d42dd25cee2c Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 17 Jul 2024 12:16:52 -0400 Subject: [PATCH 104/146] enforce field quoting --- examples/client/file_hierarchy_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index d7e8992d..e9e79418 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -193,7 +193,7 @@ def write_output_file(version_report, output_file): logging.info(f"Writing CSV output into {output_file}") field_names = get_csv_fieldnames() with open(output_file, "w") as f: - writer = csv.DictWriter(f, fieldnames = field_names, extrasaction = 'ignore') # TODO + writer = csv.DictWriter(f, fieldnames = field_names, extrasaction = 'ignore',quoting=csv.QUOTE_ALL) # TODO writer.writeheader() writer.writerows(get_csv_data(version_report)) return From 2ba8cd5b3b0456c4ac3ea229087c7ccc3dfdc2a0 Mon Sep 17 00:00:00 2001 From: dnichol Date: Thu, 18 Jul 2024 13:49:21 +0100 Subject: [PATCH 105/146] Remove file path column, remove duplicate components from components report --- examples/client/file_hierarchy_report.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index e9e79418..64be9f46 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -170,22 +170,28 @@ def trim_version_report(version_report, reduced_path_set): ''' def get_csv_fieldnames(): - return ['component name', 'version name', 'license', 'file path', 'match type', 'review status'] + return ['component name', 'version name', 'license', 'match type', 'review status'] def get_csv_data(version_report): csv_data = list() + components = list() for bom_view_entry in version_report['aggregateBomViewEntries']: - entry = dict() + entry = dict() entry['component name'] = bom_view_entry['producerProject']['name'] entry['version name'] = bom_view_entry['producerReleases'][0]['version'] entry['license'] = bom_view_entry['licenses'][0]['licenseDisplay'].replace(' AND ',';').replace('(','').replace(')','') pid = bom_view_entry['producerProject']['id'] vid = bom_view_entry['producerReleases'][0]['id'] - path_list = [p['path'] for p in version_report['detailedFileBomViewEntries'] if p['projectId'] == pid and p['versionId'] == vid] - entry['file path'] = ';'.join(path_list) + #path_list = [p['path'] for p in version_report['detailedFileBomViewEntries'] if p['projectId'] == pid and p['versionId'] == vid] + #entry['file path'] = ';'.join(path_list) entry['match type'] = ';'.join(bom_view_entry['matchTypes']) entry['review status'] = bom_view_entry['reviewSummary']['reviewStatus'] - csv_data.append(entry) + + # Only add if this component was not previously added. + composite_key = pid + vid + if composite_key not in components: + csv_data.append(entry) + components.append(composite_key) return csv_data def write_output_file(version_report, output_file): From a725552a1c00e4f27c3fd50efd933fc1e0afb02f Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 18 Jul 2024 13:30:35 -0400 Subject: [PATCH 106/146] csv de-duplication added --- examples/client/file_hierarchy_report.py | 40 ++++++++++++++++++++---- 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 64be9f46..5a67ec0f 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -172,7 +172,7 @@ def trim_version_report(version_report, reduced_path_set): def get_csv_fieldnames(): return ['component name', 'version name', 'license', 'match type', 'review status'] -def get_csv_data(version_report): +def get_csv_data(version_report, keep_dupes): csv_data = list() components = list() for bom_view_entry in version_report['aggregateBomViewEntries']: @@ -192,16 +192,43 @@ def get_csv_data(version_report): if composite_key not in components: csv_data.append(entry) components.append(composite_key) - return csv_data - -def write_output_file(version_report, output_file): + if keep_dupes: + return csv_data + else: + return remove_duplicates(csv_data) + +def remove_duplicates(data): + # Put data into buckets by version + buckets = dict() + for row in data: + name = row['component name'].lower() + version = row['version name'] + if not version in buckets: + buckets[version] = [row] + else: + buckets[version].append(row) + # Run reduction process in component names that start with existing component name + # This process will ignore case in component names + for set in buckets.values(): + set.sort(key = lambda d: d['component name'].lower()) + for row in set: + index = set.index(row) + name = row['component name'].lower() + while index + 1 < len(set) and set[index+1]['component name'].lower().startswith(name): + set.pop(index+1) + reduced_data = list() + for b in buckets.values(): + reduced_data.extend(b) + return reduced_data + +def write_output_file(version_report, output_file, keep_dupes): if output_file.lower().endswith(".csv"): logging.info(f"Writing CSV output into {output_file}") field_names = get_csv_fieldnames() with open(output_file, "w") as f: writer = csv.DictWriter(f, fieldnames = field_names, extrasaction = 'ignore',quoting=csv.QUOTE_ALL) # TODO writer.writeheader() - writer.writerows(get_csv_data(version_report)) + writer.writerows(get_csv_data(version_report, keep_dupes)) return # If it's neither, then .json if not output_file.lower().endswith(".json"): @@ -219,6 +246,7 @@ def parse_command_args(): parser.add_argument("-pn", "--project-name", required=True, help="Project Name") parser.add_argument("-pv", "--project-version-name", required=True, help="Project Version Name") parser.add_argument("-o", "--output-file", required=False, help="File name to write output. File extension determines format .json and .csv, json is the default.") + parser.add_argument("-kd", "--keep-dupes", action='store_true', help="Do not reduce CVS data by fuzzy matching component names") parser.add_argument("-kh", "--keep_hierarchy", action='store_true', help="Set to keep all entries in the sources report. Will not remove components found under others.") parser.add_argument("--report-retries", metavar="", type=int, default=RETRY_LIMIT, help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") parser.add_argument("--report-timeout", metavar="", type=int, default=RETRY_TIMER, help="Wait time between subsequent download attempts.") @@ -266,7 +294,7 @@ def main(): trim_version_report(version_report, reduced_path_set) logging.info(f"Truncated dataset contains {len(version_report['aggregateBomViewEntries'])} bom entries and {len(version_report['detailedFileBomViewEntries'])} file view entries") - write_output_file(version_report, output_file) + write_output_file(version_report, output_file, args.keep_dupes) # Combine component data with selected file data # Output result with CSV anf JSON as options. From 084b557cfc2f6c63d32faa994433bf6182639824 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 18 Jul 2024 13:39:15 -0400 Subject: [PATCH 107/146] csv de-duplication added --- examples/client/file_hierarchy_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py index 5a67ec0f..d80f1d45 100644 --- a/examples/client/file_hierarchy_report.py +++ b/examples/client/file_hierarchy_report.py @@ -207,7 +207,7 @@ def remove_duplicates(data): buckets[version] = [row] else: buckets[version].append(row) - # Run reduction process in component names that start with existing component name + # Run reduction process for component names that start with existing component name # This process will ignore case in component names for set in buckets.values(): set.sort(key = lambda d: d['component name'].lower()) From bcd8d56701497590f26c3b729cad4da89d6fb233 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 31 Jul 2024 10:49:20 -0400 Subject: [PATCH 108/146] consolidated log tracking --- examples/client/multi-image/generate-clone.sh | 5 +- .../multi-image/manage_project_structure.py | 667 ++++++++++-------- 2 files changed, 363 insertions(+), 309 deletions(-) diff --git a/examples/client/multi-image/generate-clone.sh b/examples/client/multi-image/generate-clone.sh index c968f0d6..d577d216 100644 --- a/examples/client/multi-image/generate-clone.sh +++ b/examples/client/multi-image/generate-clone.sh @@ -9,7 +9,9 @@ le:testcontainer:2.4,\ login-app-ui:testcontainer:2.4,\ nlv:testcontainer:2.4" -SPECFILE=~/Documents/Ciena/excelparameters/BP_SampleProduct.xlsx +# SPECFILE=~/Documents/Ciena/excelparameters/BP_SampleProduct.xlsx +SPECFILE=~/Documents/Ciena/excelparameters/BP_SampleProduct_Truncated.xlsx +TEXTFILE=~/Documents/Ciena/excelparameters/bdscaninput.txt ls -l $SPECFILE @@ -17,4 +19,5 @@ COMMAND="python3 examples/client/multi-image/manage_project_structure.py" # $COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ $COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@ +# $COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $TEXTFILE --clone-from 2.3 $@ diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 99ca89dd..1ecbc13c 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -20,65 +20,39 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +''' +program_description = \ +''' This script will scan a multi-container project into hierarchical structure. Project Name Project Version - Subproject Name - Subproject Vesrsion + Sub-project Name + Sub-project Version Base Image Base Image Version Add on Image Add on Image Version -usage: python3 manage_project_structure.py [-h] -u BASE_URL -t TOKEN_FILE [-pg PROJECT_GROUP] -p PROJECT_NAME -pv VERSION_NAME - [-sp SUBPROJECT_LIST | -ssf SUBPROJECT_SPEC_FILE] [-nv] [-rm] [--clone-from CLONE_FROM] - [--dry-run] [-str STRING_TO_PUT_IN_FRONT_OF_SUBPROJECT_NAME] - -options: - -h, --help show this help message and exit - -u BASE_URL, --base-url BASE_URL - Hub server URL e.g. https://your.blackduck.url - -t TOKEN_FILE, --token-file TOKEN_FILE - File containing access token - -pg PROJECT_GROUP, --project_group PROJECT_GROUP - Project Group to be used - -p PROJECT_NAME, --project-name PROJECT_NAME - Project Name - -pv VERSION_NAME, --version-name VERSION_NAME - Project Version Name - -sp SUBPROJECT_LIST, --subproject-list SUBPROJECT_LIST - List of subprojects to generate with subproject:container:tag - -ssf SUBPROJECT_SPEC_FILE, --subproject-spec-file SUBPROJECT_SPEC_FILE - Excel or txt file containing subproject specification - -nv, --no-verify Disable TLS certificate verification - -rm, --remove Remove project structure with all subprojects (DANGEROUS!) - --clone-from CLONE_FROM - Main project version to use as template for cloning - --dry-run Create structure only, do not execute scans - -str STRING_TO_PUT_IN_FRONT_OF_SUBPROJECT_NAME, --string-to-put-in-front-of-subproject-name STRING_TO_PUT_IN_FRONT_OF_SUBPROJECT_NAME - -Subprojects ae specified as subproject:[container]:[tag] -if container name omited it will be set to subproject -if tag omited it would be set to 'latest' - -Subprojects an be specified in excel file with -ssf --subproject-spec-file parameter. +Sub-projects are specified as sub-project:[container]:[tag] +if container name omitted it will be set to sub-project +if tag omitted it would be set to 'latest' + +Sub-projects an be specified in excel file with -ssf --subproject-spec-file parameter. Excel file should contain one worksheet with first row containing column names as following: Container Name, Image ID, Version, Project Name and subsequent rows containing data -Sublrojects could be specified in a text file with -ssf --subproject-spec-file parameter +Sub-projects could be specified in a text file with -ssf --subproject-spec-file parameter Each line will have to contain full image specification. -Specification will be parsed and image name prefixed by -str parameter will be used as a subproject name +Specification will be parsed and image name prefixed by -str parameter will be used as a sub-project name In this mode any image that is not residing on ciena.com repository will be skipped. If -str parameter is empty, Project Name will be used instead. Container image name scanned will be written into project version nickname field - - ''' import argparse @@ -90,289 +64,359 @@ from blackduck import Client from pprint import pprint,pformat -logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) -logging.getLogger("requests").setLevel(logging.INFO) -logging.getLogger("urllib3").setLevel(logging.INFO) -logging.getLogger("blackduck").setLevel(logging.INFO) - -strict = False - -def remove_project_structure(project_name, version_name): - project = find_project_by_name(project_name) - if not project: - logging.info(f"Project {project_name} does not exist.") - return - num_versions = bd.get_resource('versions', project, items=False)['totalCount'] - version = find_project_version_by_name(project,version_name) - if not version: - logging.info(f"Project {project_name} with version {version_name} does not exist.") - return - components = [ - c for c in bd.get_resource('components',version) if c['componentType'] == "SUB_PROJECT" - ] - logging.info(f"Project {project_name}:{version_name} has {len(components)} subprojects") - for component in components: - component_name = component['componentName'] - component_version_name = component['componentVersionName'] - logging.info(f"Removing subproject {component_name} from {project_name}:{version_name}") - component_url = component['_meta']['href'] - response = bd.session.delete(component_url) - logging.info(f"Operation completed with {response}") - remove_project_structure(component_name, component_version_name) - logging.info(f"Removing {project_name}:{version_name}") - if num_versions > 1: - response = bd.session.delete(version['_meta']['href']) - else: - response = bd.session.delete(project['_meta']['href']) - logging.info(f"Operation completed with {response}") - -def remove_codelocations_recursively(version): - components = bd.get_resource('components', version) - subprojects = [x for x in components if x['componentType'] == 'SUB_PROJECT'] - logging.info(f"Found {len(subprojects)} subprojects") - unmap_all_codelocations(version) - for subproject in subprojects: - subproject_name = subproject['componentName'] - subproject_version_name = subproject['componentVersionName'] - project = find_project_by_name(subproject_name) +class MultiImageProjectManager(): + + def __init__(self, args): + self.debug = args.debug + self.log_config() + self.base_url = args.base_url + with open(args.token_file, 'r') as tf: + self.access_token = tf.readline().strip() + self.no_verify = args.no_verify + self.connect() + self.init_project_data(args) + + def connect(self): + self.client = Client(base_url=self.base_url, token=self.access_token, verify=self.no_verify, timeout=60.0, retries=4) + + def init_project_data(self,args): + self.project_data = dict() + self.project_data['project_name'] = args.project_name + self.project_data['version_name'] = args.version_name + self.project_data['project_group'] = args.project_group + self.project_data['clone_from'] = args.clone_from + self.project_data['dry_run'] = args.dry_run + self.project_data['strict'] = args.strict + self.project_data['remove'] = args.remove + + subprojects = dict() + child_spec_list = self.get_child_spec_list(args) + for child_spec in [x.split(':') for x in child_spec_list]: + subproject = dict() + i = iter(child_spec) + i = iter(child_spec) + child = next(i) + repo = next(i, child) + tag = next(i,'latest') + container_spec = f"{repo}:{tag}" + while child in subprojects: + child += "_" + subproject['image'] = container_spec + subproject['project_name'] = child + subproject['version_name'] = args.version_name + subproject['project_group'] = args.project_group + subproject['clone_from'] = args.clone_from + subprojects[child] = subproject + + self.project_data['subprojects'] = subprojects + + def log_config(self): + if self.debug: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.DEBUG) + else: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.INFO) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("blackduck").setLevel(logging.WARNING) + + def log(self, level, msg, target=None): + if target: + target_log = target.get('log', None) + if not target_log: + target_log = list() + target['log'] = target_log + target_log.append(msg) + log_function = getattr(logging, level) + log_function(msg) + + def remove_project_structure(self, project_name, version_name): + project = self.find_project_by_name(project_name) + if not project: - logging.info(f"Project {subproject_name} does not exist.") + logging.info(f"Project {project_name} does not exist.") return - subproject_version = find_project_version_by_name(project, subproject_version_name) - if not subproject_version: - logging.info(f"Project {subproject_name} with version {subversion_name} does not exist.") + num_versions = self.client.get_resource('versions', project, items=False)['totalCount'] + version = self.find_project_version_by_name(project,version_name) + if not version: + logging.info(f"Project {project_name} with version {version_name} does not exist.") return - remove_codelocations_recursively(subproject_version) - -def unmap_all_codelocations(version): - codelocations = bd.get_resource('codelocations',version) - for codelocation in codelocations: - logging.info(f"Unmapping codelocation {codelocation['name']}") - codelocation['mappedProjectVersion'] = "" - response = bd.session.put(codelocation['_meta']['href'], json=codelocation) - pprint (response) - - -def find_or_create_project_group(group_name): - url = '/api/project-groups' - params = { - 'q': [f"name:{group_name}"] - } - groups = [p for p in bd.get_items(url, params=params) if p['name'] == group_name] - if len(groups) == 0: - headers = { - 'Accept': 'application/vnd.blackducksoftware.project-detail-5+json', - 'Content-Type': 'application/vnd.blackducksoftware.project-detail-5+json' + components = [ + c for c in self.client.get_resource('components',version) if c['componentType'] == "SUB_PROJECT" + ] + logging.info(f"Project {project_name}:{version_name} has {len(components)} subprojects") + for component in components: + component_name = component['componentName'] + component_version_name = component['componentVersionName'] + logging.info(f"Removing subproject {component_name} from {project_name}:{version_name}") + component_url = component['_meta']['href'] + response = self.client.session.delete(component_url) + logging.info(f"Operation completed with {response}") + self.remove_project_structure(component_name, component_version_name) + logging.info(f"Removing {project_name}:{version_name}") + if num_versions > 1: + response = self.client.session.delete(version['_meta']['href']) + else: + response = self.client.session.delete(project['_meta']['href']) + logging.info(f"Operation completed with {response}") + + def remove_codelocations_recursively(version): + components = self.client.get_resource('components', version) + subprojects = [x for x in components if x['componentType'] == 'SUB_PROJECT'] + logging.info(f"Found {len(subprojects)} subprojects") + unmap_all_codelocations(version) + for subproject in subprojects: + subproject_name = subproject['componentName'] + subproject_version_name = subproject['componentVersionName'] + project = find_project_by_name(subproject_name) + if not project: + logging.info(f"Project {subproject_name} does not exist.") + return + subproject_version = find_project_version_by_name(project, subproject_version_name) + if not subproject_version: + logging.info(f"Project {subproject_name} with version {subversion_name} does not exist.") + return + remove_codelocations_recursively(subproject_version) + + def unmap_all_codelocations(version): + codelocations = self.client.get_resource('codelocations',version) + for codelocation in codelocations: + logging.info(f"Unmapping codelocation {codelocation['name']}") + codelocation['mappedProjectVersion'] = "" + response = self.client.session.put(codelocation['_meta']['href'], json=codelocation) + pprint (response) + + + def find_or_create_project_group(self, group_name): + url = '/api/project-groups' + params = { + 'q': [f"name:{group_name}"] } - data = { - 'name': group_name + groups = [p for p in self.client.get_items(url, params=params) if p['name'] == group_name] + if len(groups) == 0: + headers = { + 'Accept': 'application/vnd.blackducksoftware.project-detail-5+json', + 'Content-Type': 'application/vnd.blackducksoftware.project-detail-5+json' + } + data = { + 'name': group_name + } + response = self.client.session.post(url, headers=headers, json=data) + return response.headers['Location'] + else: + return groups[0]['_meta']['href'] + + def find_project_by_name(self, project_name): + params = { + 'q': [f"name:{project_name}"] } - response = bd.session.post(url, headers=headers, json=data) - return response.headers['Location'] - else: - return groups[0]['_meta']['href'] - -def find_project_by_name(project_name): - params = { - 'q': [f"name:{project_name}"] - } - projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] - if len(projects) == 1: - return projects[0] - else: - return None - -def find_project_version_by_name(project, version_name): - params = { - 'q': [f"versionName:{version_name}"] - } - versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] - if len(versions) == 1: - return versions[0] - else: - return None - -def create_project_version(project_name,version_name,args, nickname = None): - version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name} - if nickname: - version_data['nickname'] = nickname - url = '/api/projects' - project = find_project_by_name(project_name) - if project: - data = version_data - url = project['_meta']['href'] + '/versions' - else: - data = {"name": project_name, - "projectGroup": find_or_create_project_group(args.project_group), - "versionRequest": version_data} - return bd.session.post(url, json=data) - -def add_component_to_version_bom(child_version, version): - url = version['_meta']['href'] + '/components' - data = { 'component': child_version['_meta']['href']} - return bd.session.post(url, json=data) - -def process_excel_spec_file(wb): - ws = wb.active - project_list = [] - row_number = 0 - for row in ws.values: - row_number += 1 - if (row_number == 1 and - row[0] == 'Container Name' and - row[1] == 'Image ID' and - row[2] == 'Version' and - row[3] == 'Project Name'): - logging.info(f"File Format checks out (kind of)") - continue - elif row_number > 1: - project_list.append(f"{row[3]}:{row[0]}:{row[2]}") + projects = [p for p in self.client.get_resource('projects', params=params) if p['name'] == project_name] + if len(projects) == 1: + return projects[0] else: - logging.error(f"Could not parse input file {args.subproject_spec_file}") - sys.exit(1) - return (project_list) - -def process_text_spec_file(args): - project_list = [] - prefix = args.string_to_put_in_front_of_subproject_name - if not prefix: - prefix = args.project_name - with open(args.subproject_spec_file, "r") as f: - lines = f.read().splitlines() - for line in lines: - image_name = line.split('/')[-1].split(':')[0] # Don't look at me, you wrote it! - sub_project_name = "_".join((prefix, image_name)) - spec_line = ":".join((sub_project_name, line)) - if "ciena.com" in spec_line: - project_list.append(spec_line) - return (project_list) - -def get_child_spec_list(args): - if args.subproject_list: - return args.subproject_list.split(',') - else: - # Excel and plaintext - logging.info(f"Processing excel file {args.subproject_spec_file}") - import openpyxl - try: - wb = openpyxl.load_workbook(args.subproject_spec_file) - return process_excel_spec_file(wb) - except Exception: - return process_text_spec_file(args) - -def create_and_add_child_projects(version, args): - version_url = version['_meta']['href'] + '/components' - child_spec_list = get_child_spec_list(args) - for child_spec in [x.split(':') for x in child_spec_list]: - i = iter(child_spec) - child = next(i) - repo = next(i, child) - tag = next(i,'latest') - container_spec = f"{repo}:{tag}" - scan_param = {'image': container_spec, 'project': child, 'version': args.version_name, 'project_group': args.project_group} - if args.clone_from: - scan_param['clone_from'] = args.clone_from - project = find_project_by_name(child) + return None + + def find_project_version_by_name(self, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in self.client.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + + def create_project_version(self,project_name,version_name,nickname = None): + version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name} + if nickname: + version_data['nickname'] = nickname + url = '/api/projects' + project = self.find_project_by_name(project_name) if project: - version = find_project_version_by_name(project,args.version_name) - if version: - if strict: - logging.error(f"Child project {project['name']} with version {args.version_name} exists.") - sys.exit(1) + data = version_data + url = project['_meta']['href'] + '/versions' + else: + data = {"name": project_name, + "projectGroup": self.find_or_create_project_group(self.project_data['project_group']), + "versionRequest": version_data} + return self.client.session.post(url, json=data) + + def add_component_to_version_bom(child_version, version): + url = version['_meta']['href'] + '/components' + data = { 'component': child_version['_meta']['href']} + return self.client.session.post(url, json=data) + + def process_excel_spec_file(self,wb): + ws = wb.active + project_list = [] + row_number = 0 + for row in ws.values: + row_number += 1 + if (row_number == 1 and + row[0] == 'Container Name' and + row[1] == 'Image ID' and + row[2] == 'Version' and + row[3] == 'Project Name'): + logging.info(f"File Format checks out (kind of)") + continue + elif row_number > 1: + project_list.append(f"{row[3]}:{row[0]}:{row[2]}") + else: + logging.error(f"Could not parse input file ") + sys.exit(1) + return (project_list) + + def process_text_spec_file(self,args): + project_list = [] + prefix = args.string_to_put_in_front_of_subproject_name + if not prefix: + prefix = args.project_name + with open(args.subproject_spec_file, "r") as f: + lines = f.read().splitlines() + for line in lines: + image_name = line.split('/')[-1].split(':')[0] # Don't look at me, you wrote it! + sub_project_name = "_".join((prefix, image_name)) + spec_line = ":".join((sub_project_name, line)) + if "ciena.com" in spec_line: + project_list.append(spec_line) + return (project_list) + + def get_child_spec_list(self,args): + if args.subproject_list: + return args.subproject_list.split(',') + else: + # Excel and plaintext + logging.debug(f"Processing excel file {args.subproject_spec_file}") + import openpyxl + try: + wb = openpyxl.load_workbook(args.subproject_spec_file) + return self.process_excel_spec_file(wb) + except Exception: + return self.process_text_spec_file(args) + + def create_and_add_child_projects(self,version): + version_url = version['_meta']['href'] + '/components' + version_name = self.project_data['version_name'] + for child_name, child in self.project_data['subprojects'].items(): + child_name = child['project_name'] + container_spec = child['image'] + project = self.find_project_by_name(child_name) + if project: + version = self.find_project_version_by_name(project,version_name) + if version: + if self.project_data['strict']: + self.log('error', f"Child project {project['name']} with version {version_name} exists.", child) + sys.exit(1) + else: + self.log('debug',f"Child project {project['name']} with version {version_name} found.", child) + self.log('debug',f"Recursively removing codelocations for {project['name']} with version {version_name} ", child) + try: + self.log('debug',f"Adding project {child_name} {version_name} to the parent project", child) + child_version_url = version['_meta']['href'] + response = self.client.session.post(version_url,json={'component': child_version_url}) + self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with {response}", child) + except Exception as e: + self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with exception {e}", child) + + # remove_codelocations_recursively(version) else: - logging.info(f"Child project {project['name']} with version {args.version_name} found.") - logging.info(f"Recursively removing codelocations for {project['name']} with version {args.version_name} ") - try: - logging.info(f"Adding project {child} {args.version_name} to the parent project") - child_version_url = version['_meta']['href'] - response = bd.session.post(version_url,json={'component': child_version_url}) - logging.info(f"Adding {child} : {args.version_name} to parent project completed with {response}") - except Exception as e: - logging.info(f"Adding {child} : {args.version_name} to parent project completed with exception {e}") - - remove_codelocations_recursively(version) + response = self.create_project_version(child_name,version_name, nickname=container_spec) + self.log('debug',f"Creating project {child_name} : {version_name} completed with {response}", child) + if response.ok: + child_version = self.find_project_version_by_name(self.find_project_by_name(child_name),version_name) + child_version_url = child_version['_meta']['href'] + response = self.client.session.post(version_url,json={'component': child_version_url}) + self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with {response}", child) else: - response = create_project_version(child,args.version_name, args, nickname=container_spec) - logging.info(f"Creating project {child} : {args.version_name} completed with {response}") + response = self.create_project_version(child_name,version_name, nickname=container_spec) + self.log('debug',f"Creating project {child_name} : {version_name} completed with {response}",child) if response.ok: - child_version = find_project_version_by_name(find_project_by_name(child),args.version_name) + child_version = self.find_project_version_by_name(self.find_project_by_name(child_name),version_name) child_version_url = child_version['_meta']['href'] - response = bd.session.post(version_url,json={'component': child_version_url}) - logging.info(f"Adding {child} : {args.version_name} to parent project completed with {response}") - else: - response = create_project_version(child,args.version_name, args, nickname=container_spec) - logging.info(f"Creating project {child} : {args.version_name} completed with {response}") - if response.ok: - child_version = find_project_version_by_name(find_project_by_name(child),args.version_name) - child_version_url = child_version['_meta']['href'] - response = bd.session.post(version_url,json={'component': child_version_url}) - logging.info(f"Adding {child} : {args.version_name} to parent project completed with {response}") - scan_params.append(scan_param) - -def create_project_structure(args): - project = find_project_by_name(args.project_name) - logging.info(f"Project {args.project_name} located") - if project: - version = find_project_version_by_name(project,args.version_name) - if version: - if strict: - logging.error(f"Project {project['name']} with version {args.version_name} exists.") - sys.exit(1) + response = self.client.session.post(version_url,json={'component': child_version_url}) + self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with {response}", child) + + def create_project_structures(self): + project_name = self.project_data['project_name'] + version_name = self.project_data['version_name'] + + project = self.find_project_by_name(project_name) + if project: + self.log('debug', f"Project {project_name} located", self.project_data) + version = self.find_project_version_by_name(project,version_name) + if version: + if self.project_data['strict']: + self.log('error', f"Project {project['name']} with version {version_name} exists.", self.project_data) + sys.exit(1) + else: + self.log('debug',f"Found Project {project['name']} with version {version_name}.", self.project_data) else: - logging.info(f"Found Project {project['name']} with version {args.version_name}.") + response = self.create_project_version(project_name,version_name) + if response.ok: + version = self.find_project_version_by_name(self.find_project_by_name(project_name),version_name) + self.log('debug', f"Project {project_name} : {version_name} created", self.project_data) + else: + self.log('debug',f"Failed to create Project {project_name} : {version_name} created", self.project_data) + sys.exit(1) else: - response = create_project_version(args.project_name,args.version_name,args) + self.log('debug',f"Project {project_name} was not found, creating ...", self.project_data) + response = self.create_project_version(project_name,version_name) if response.ok: - version = find_project_version_by_name(find_project_by_name(args.project_name),args.version_name) - logging.info(f"Project {args.project_name} : {args.version_name} created") + version = self.find_project_version_by_name(self.find_project_by_name(project_name),version_name) + self.log('debug',f"Project {project_name} : {version_name} created", self.project_data) else: - logging.info(f"Failed to create Project {args.project_name} : {args.version_name} created") + self.log('debug',f"Failed to create Project {project_name} : {version_name} created", self.project_data) sys.exit(1) - else: - response = create_project_version(args.project_name,args.version_name,args) - if response.ok: - version = find_project_version_by_name(find_project_by_name(args.project_name),args.version_name) - logging.info(f"Project {args.project_name} : {args.version_name} created") + self.log('debug',f"Checking/Adding subprojects to {project_name} : {version['versionName']}", self.project_data) + self.create_and_add_child_projects(version) + + def scan_container_images(self): + from scan_docker_image_lite import scan_container_image + from blackduck.HubRestApi import HubInstance + hub = HubInstance(self.base_url, api_token=self.access_token, insecure= not self.no_verify, debug=self.debug) + + for child_name, child in self.project_data['subprojects'].items(): + parent_project = child['project_name'] + parent_version = child['version_name'] + image_name = child['image'] + clone_from = child['clone_from'] + project_group = child['project_group'] + detect_options = (f"--detect.parent.project.name={parent_project} " + f"--detect.parent.project.version.name={parent_version} " + f"--detect.project.version.nickname={image_name}") + if clone_from: + detect_options += f" --detect.clone.project.version.name={clone_from}" + if project_group: + detect_options += f" --detect.project.group.name=\"{project_group}\"" + try: + scan_container_image( + image_name, + None, + None, + None, + parent_project, + parent_version, + detect_options, + hub=hub, + binary=False + ) + except Exception: + import traceback + traceback.print_exc() + logging.error(f"Scanning of {image_name} failed, skipping") + + def proceed(self): + if self.project_data['remove']: + project_name = self.project_data['project_name'] + version_name = self.project_data['version_name'] + self.remove_project_structure(project_name, version_name) else: - logging.info(f"Failed to create Project {args.project_name} : {args.version_name} created") - sys.exit(1) - logging.info(f"Checking/Adding subprojects to {args.project_name} : {version['versionName']}") - create_and_add_child_projects(version, args) - -def scan_container_images(scan_params, hub): - from scan_docker_image_lite import scan_container_image - for params in scan_params: - detect_options = (f"--detect.parent.project.name={params['project']} " - f"--detect.parent.project.version.name={params['version']} " - f"--detect.project.version.nickname={params['image']}") - clone_from = params.get('clone_from', None) - if clone_from: - detect_options += f" --detect.clone.project.version.name={clone_from}" - project_group = params.get('project_group', None) - if project_group: - detect_options += f" --detect.project.group.name=\"{project_group}\"" - try: - scan_container_image( - params['image'], - None, - None, - None, - params['project'], - params['version'], - detect_options, - hub=hub, - binary=False - ) - except Exception: - import traceback - traceback.print_exc() - logging.error(f"Scanning of {params['image']} failed, skipping") - skipped_scans.append(params) - + self.create_project_structures() + self.scan_container_images() + def parse_command_args(): - parser = argparse.ArgumentParser("python3 manage_project_structure.py") + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") parser.add_argument("-t", "--token-file", required=True, help="File containing access token") parser.add_argument("-pg", "--project_group", required=False, default='Multi-Image', help="Project Group to be used") @@ -386,23 +430,30 @@ def parse_command_args(): parser.add_argument("--clone-from", required=False, help="Main project version to use as template for cloning") parser.add_argument("--dry-run", action='store_true', required=False, help="Create structure only, do not execute scans") parser.add_argument("-str", "--string-to-put-in-front-of-subproject-name", required=False, help="Prefix string for subproject names" ) + parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") + parser.add_argument("--strict", action='store_true', help="Fail if existing (sub)project versions already exist") return parser.parse_args() def main(): args = parse_command_args() - with open(args.token_file, 'r') as tf: - access_token = tf.readline().strip() - global bd + mipm = MultiImageProjectManager(args) + logging.info(f"Parsed {len(mipm.project_data['subprojects'])} projects from specification data") + mipm.proceed() + pprint(mipm.project_data) + sys.exit(1) + + log_config(args.debug) global scan_params, skipped_scans scan_params = [] skipped_scans = [] - bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) - logging.info(f"{args}") + logging.debug(f"{args}") + structure = define_project_structure(args) if args.remove: - remove_project_structure(args.project_name, args.version_name) + remove_project_structure(args.project_name, args.version_name, structure) else: - create_project_structure(args) + create_project_structure(structure) + sys.exit(10) if args.dry_run: logging.info(f"{pformat(scan_params)}") else: From 391b0303bf4d0c353885e8258a9c4742b1c8d71b Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 2 Aug 2024 12:39:48 -0400 Subject: [PATCH 109/146] Centralized execution results implemented --- .../multi-image/manage_project_structure.py | 91 +++++++++---------- .../multi-image/scan_docker_image_lite.py | 27 ++++-- 2 files changed, 61 insertions(+), 57 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 1ecbc13c..14bd58d5 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -91,6 +91,7 @@ def init_project_data(self,args): subprojects = dict() child_spec_list = self.get_child_spec_list(args) + # pprint (child_spec_list) for child_spec in [x.split(':') for x in child_spec_list]: subproject = dict() i = iter(child_spec) @@ -133,57 +134,57 @@ def remove_project_structure(self, project_name, version_name): project = self.find_project_by_name(project_name) if not project: - logging.info(f"Project {project_name} does not exist.") + logging.debug(f"Project {project_name} does not exist.") return num_versions = self.client.get_resource('versions', project, items=False)['totalCount'] version = self.find_project_version_by_name(project,version_name) if not version: - logging.info(f"Project {project_name} with version {version_name} does not exist.") + logging.debug(f"Project {project_name} with version {version_name} does not exist.") return components = [ c for c in self.client.get_resource('components',version) if c['componentType'] == "SUB_PROJECT" ] - logging.info(f"Project {project_name}:{version_name} has {len(components)} subprojects") + logging.debug(f"Project {project_name}:{version_name} has {len(components)} subprojects") for component in components: component_name = component['componentName'] component_version_name = component['componentVersionName'] - logging.info(f"Removing subproject {component_name} from {project_name}:{version_name}") + logging.debug(f"Removing subproject {component_name} from {project_name}:{version_name}") component_url = component['_meta']['href'] response = self.client.session.delete(component_url) - logging.info(f"Operation completed with {response}") + logging.debug(f"Operation completed with {response}") self.remove_project_structure(component_name, component_version_name) - logging.info(f"Removing {project_name}:{version_name}") + logging.debug(f"Removing {project_name}:{version_name}") if num_versions > 1: response = self.client.session.delete(version['_meta']['href']) else: response = self.client.session.delete(project['_meta']['href']) - logging.info(f"Operation completed with {response}") + logging.debug(f"Operation completed with {response}") - def remove_codelocations_recursively(version): + def remove_codelocations_recursively(self, version): components = self.client.get_resource('components', version) subprojects = [x for x in components if x['componentType'] == 'SUB_PROJECT'] - logging.info(f"Found {len(subprojects)} subprojects") - unmap_all_codelocations(version) + logging.debug(f"Found {len(subprojects)} subprojects") + self.unmap_all_codelocations(version) for subproject in subprojects: subproject_name = subproject['componentName'] subproject_version_name = subproject['componentVersionName'] - project = find_project_by_name(subproject_name) + project = self.find_project_by_name(subproject_name) if not project: - logging.info(f"Project {subproject_name} does not exist.") + logging.debug(f"Project {subproject_name} does not exist.") return - subproject_version = find_project_version_by_name(project, subproject_version_name) + subproject_version = self.find_project_version_by_name(project, subproject_version_name) if not subproject_version: - logging.info(f"Project {subproject_name} with version {subversion_name} does not exist.") + logging.debug(f"Project {subproject_name} with version {subproject_version_name} does not exist.") return - remove_codelocations_recursively(subproject_version) + self.remove_codelocations_recursively(subproject_version) - def unmap_all_codelocations(version): + def unmap_all_codelocations(self, version): codelocations = self.client.get_resource('codelocations',version) for codelocation in codelocations: - logging.info(f"Unmapping codelocation {codelocation['name']}") + logging.debug(f"Un-mapping of code location {codelocation['name']}") codelocation['mappedProjectVersion'] = "" response = self.client.session.put(codelocation['_meta']['href'], json=codelocation) - pprint (response) + logging.debug(f"Un-mapping of code location {codelocation['name']} completed with {response}") def find_or_create_project_group(self, group_name): @@ -316,8 +317,7 @@ def create_and_add_child_projects(self,version): self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with {response}", child) except Exception as e: self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with exception {e}", child) - - # remove_codelocations_recursively(version) + self.remove_codelocations_recursively(version) else: response = self.create_project_version(child_name,version_name, nickname=container_spec) self.log('debug',f"Creating project {child_name} : {version_name} completed with {response}", child) @@ -335,7 +335,7 @@ def create_and_add_child_projects(self,version): response = self.client.session.post(version_url,json={'component': child_version_url}) self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with {response}", child) - def create_project_structures(self): + def create_project_structure(self): project_name = self.project_data['project_name'] version_name = self.project_data['version_name'] @@ -369,6 +369,21 @@ def create_project_structures(self): self.log('debug',f"Checking/Adding subprojects to {project_name} : {version['versionName']}", self.project_data) self.create_and_add_child_projects(version) + def validate_project_structure(self): + parent_project = self.find_project_by_name(self.project_data['project_name']) + parent_version = self.find_project_version_by_name(parent_project, self.project_data['version_name']) + components = [c for c in self.client.get_resource('components', parent_version) if c['componentType'] == 'SUB_PROJECT'] + for spn, sp in self.project_data['subprojects'].items(): + spn = sp['project_name'] + spvn = sp['version_name'] + ca = [c for c in components if spn == c['componentName'] and spvn == c['componentVersionName']] + if len(ca) == 1: + sp['status'] = 'PRESENT' + self.log('debug',f"Sub-project project {spn} : {spvn} is present in the BOM ", sp) + else: + sp['status'] = 'ABSENT' + self.log('debug',f"Sub-project project {spn} : {spvn} could not be added to the BOM ", sp) + def scan_container_images(self): from scan_docker_image_lite import scan_container_image from blackduck.HubRestApi import HubInstance @@ -388,7 +403,7 @@ def scan_container_images(self): if project_group: detect_options += f" --detect.project.group.name=\"{project_group}\"" try: - scan_container_image( + results = scan_container_image( image_name, None, None, @@ -399,6 +414,7 @@ def scan_container_images(self): hub=hub, binary=False ) + child['scan_results'] = results except Exception: import traceback traceback.print_exc() @@ -410,7 +426,8 @@ def proceed(self): version_name = self.project_data['version_name'] self.remove_project_structure(project_name, version_name) else: - self.create_project_structures() + self.create_project_structure() + self.validate_project_structure() self.scan_container_images() @@ -439,33 +456,7 @@ def main(): mipm = MultiImageProjectManager(args) logging.info(f"Parsed {len(mipm.project_data['subprojects'])} projects from specification data") mipm.proceed() - pprint(mipm.project_data) - sys.exit(1) - - log_config(args.debug) - global scan_params, skipped_scans - scan_params = [] - skipped_scans = [] - logging.debug(f"{args}") - structure = define_project_structure(args) - - if args.remove: - remove_project_structure(args.project_name, args.version_name, structure) - else: - create_project_structure(structure) - sys.exit(10) - if args.dry_run: - logging.info(f"{pformat(scan_params)}") - else: - logging.info("Now executing scans") - from blackduck.HubRestApi import HubInstance - hub = HubInstance(args.base_url, api_token=access_token, insecure=True, debug=False) - scan_container_images(scan_params, hub) - if len(skipped_scans) > 0: - logging.info(f"The following images were not scanned") - logging.info(f"{pformat(skipped_scans)}") - - + if __name__ == "__main__": sys.exit(main()) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index d43875c8..4df73e54 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -253,9 +253,14 @@ def __init__( def prepare_container_image(self): self.docker.initdir() - self.docker.pull_container_image(self.container_image_name) - self.docker.save_container_image(self.container_image_name) - self.docker.unravel_container() + result = self.docker.pull_container_image(self.container_image_name) + logging.debug(f"Command {" ".join(result.args)} exited with returncode {result.returncode}") + result = self.docker.save_container_image(self.container_image_name) + if result.returncode: + raise Exception (f"Command {" ".join(result.args)} failed with returncode {result.returncode} error = {result.stdout}") + result = self.docker.unravel_container() + if result.returncode: + raise Exception (f"Command {" ".join(result.args)} failed with returncode {result.returncode} error = {result.stdout}") # result = self.docker.get_container_image_history(self.container_image_name) history = self.docker.read_config()['history'] layer_count = 0 @@ -378,7 +383,7 @@ def get_group_name(self, groups, index): return group_name def process_oci_container_image_by_base_image_info(self): - print ("Processing by BAse Image not supported for OCI images") + print ("Processing by Base Image not supported for OCI images") sys.exit(1) pass @@ -420,10 +425,16 @@ def submit_layer_scans(self): options.extend(self.adorn_extra_options(layer)) else: options.extend(self.extra_options) - logging.info(f"Submitting scan for {layer['name']}") + logging.debug(f"Submitting scan for {layer['name']}") completed = self.hub_detect.detect_run(options) - logging.info(f"Detect run for {layer['name']} completed with returncode {completed.returncode}") - + scan_results = dict() + for key, value in vars(completed).items(): + if type(value) is bytes: + scan_results[key] = value.decode('utf-8') + else: + scan_results[key] = value + layer['scan_results'] = scan_results + logging.debug(f"Detect run for {layer['name']} completed with returncode {completed.returncode}") def adorn_extra_options(self, layer): result = list() @@ -494,9 +505,11 @@ def scan_container_image( scanner.base_layers = scanner.get_base_layers() if binary: scanner.binary = True + logging.info(f"Scanning image {imagespec}") scanner.prepare_container_image() scanner.process_container_image() scanner.submit_layer_scans() + return scanner.layers def main(argv=None): From a24429d25addd03302c31ce5a2d8a6ebe513016c Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 5 Aug 2024 13:33:43 -0400 Subject: [PATCH 110/146] full structured log complete --- .../client/multi-image/manage_project_structure.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 14bd58d5..faf73bd7 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -68,6 +68,7 @@ class MultiImageProjectManager(): def __init__(self, args): self.debug = args.debug + self.binary = args.binary self.log_config() self.base_url = args.base_url with open(args.token_file, 'r') as tf: @@ -412,7 +413,7 @@ def scan_container_images(self): parent_version, detect_options, hub=hub, - binary=False + binary=self.binary ) child['scan_results'] = results except Exception: @@ -449,14 +450,21 @@ def parse_command_args(): parser.add_argument("-str", "--string-to-put-in-front-of-subproject-name", required=False, help="Prefix string for subproject names" ) parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") parser.add_argument("--strict", action='store_true', help="Fail if existing (sub)project versions already exist") + parser.add_argument("--binary", action='store_true', help="Use binary scan for analysis") return parser.parse_args() def main(): + from datetime import datetime + timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") args = parse_command_args() mipm = MultiImageProjectManager(args) logging.info(f"Parsed {len(mipm.project_data['subprojects'])} projects from specification data") mipm.proceed() - + filename_complete = f"{args.project_name}-{args.version_name}-{timestamp}-full.json" + filename_failures = f"{args.project_name}-{args.version_name}-{timestamp}-failures.json" + # write full processing log + with open (filename_complete, "w") as f: + json.dump(mipm.project_data, f, indent=2) if __name__ == "__main__": sys.exit(main()) From 62b82cfb5197d1593061f39884eb0d4752776bc3 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 5 Aug 2024 15:09:28 -0400 Subject: [PATCH 111/146] full structured log complete --- .../multi-image/manage_project_structure.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index faf73bd7..99746708 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -416,9 +416,10 @@ def scan_container_images(self): binary=self.binary ) child['scan_results'] = results - except Exception: - import traceback - traceback.print_exc() + except Exception as e: + # import traceback + # traceback.print_exc() + self.log('error', repr(e), child) logging.error(f"Scanning of {image_name} failed, skipping") def proceed(self): @@ -466,5 +467,22 @@ def main(): with open (filename_complete, "w") as f: json.dump(mipm.project_data, f, indent=2) + failures = list() + for sname, sub in mipm.project_data['subprojects'].items(): + structure = False + runtime = False + if sub['status'] != 'PRESENT': + structure = True + if not sub.get('scan_results', None): + runtime = True + else: + rcodes = [r['scan_results']['returncode'] for r in sub['scan_results'] if r.get('scan_results', None)] + if sum(rcodes) > 0: + runtime = True + if structure or runtime: + failures.append(sub) + with open (filename_failures, "w") as f: + json.dump(failures, f, indent=2) + if __name__ == "__main__": sys.exit(main()) From c1f9fcdbf4aa5a3a4ca6fa63dc7c77b3ff5e5177 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 12 Aug 2024 09:44:59 -0400 Subject: [PATCH 112/146] removed double quotes from interpolated string for compatibility --- examples/client/multi-image/scan_docker_image_lite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index 4df73e54..e2d912ad 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -254,13 +254,13 @@ def __init__( def prepare_container_image(self): self.docker.initdir() result = self.docker.pull_container_image(self.container_image_name) - logging.debug(f"Command {" ".join(result.args)} exited with returncode {result.returncode}") + logging.debug(f"Command {' '.join(result.args)} exited with returncode {result.returncode}") result = self.docker.save_container_image(self.container_image_name) if result.returncode: - raise Exception (f"Command {" ".join(result.args)} failed with returncode {result.returncode} error = {result.stdout}") + raise Exception (f"Command {' '.join(result.args)} failed with returncode {result.returncode} error = {result.stdout}") result = self.docker.unravel_container() if result.returncode: - raise Exception (f"Command {" ".join(result.args)} failed with returncode {result.returncode} error = {result.stdout}") + raise Exception (f"Command {' '.join(result.args)} failed with returncode {result.returncode} error = {result.stdout}") # result = self.docker.get_container_image_history(self.container_image_name) history = self.docker.read_config()['history'] layer_count = 0 From 35f9b4bd8580c7872ab83d2d182309bcd0062d01 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 12 Aug 2024 16:59:03 -0400 Subject: [PATCH 113/146] individual file matcing option added --- .../multi-image/manage_project_structure.py | 48 +++++++++++-------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 99746708..e29de739 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -69,6 +69,7 @@ class MultiImageProjectManager(): def __init__(self, args): self.debug = args.debug self.binary = args.binary + self.individual_file_matching = args.individual_file_matching self.log_config() self.base_url = args.base_url with open(args.token_file, 'r') as tf: @@ -399,6 +400,8 @@ def scan_container_images(self): detect_options = (f"--detect.parent.project.name={parent_project} " f"--detect.parent.project.version.name={parent_version} " f"--detect.project.version.nickname={image_name}") + if self.individual_file_matching: + detect_options += f" --detect.blackduck.signature.scanner.individual.file.matching=ALL" if clone_from: detect_options += f" --detect.clone.project.version.name={clone_from}" if project_group: @@ -452,6 +455,7 @@ def parse_command_args(): parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") parser.add_argument("--strict", action='store_true', help="Fail if existing (sub)project versions already exist") parser.add_argument("--binary", action='store_true', help="Use binary scan for analysis") + parser.add_argument("-ifm", "--individual-file-matching", action='store_true', help="Turn Individual file matching on") return parser.parse_args() def main(): @@ -461,28 +465,30 @@ def main(): mipm = MultiImageProjectManager(args) logging.info(f"Parsed {len(mipm.project_data['subprojects'])} projects from specification data") mipm.proceed() - filename_complete = f"{args.project_name}-{args.version_name}-{timestamp}-full.json" - filename_failures = f"{args.project_name}-{args.version_name}-{timestamp}-failures.json" - # write full processing log - with open (filename_complete, "w") as f: - json.dump(mipm.project_data, f, indent=2) - - failures = list() - for sname, sub in mipm.project_data['subprojects'].items(): - structure = False - runtime = False - if sub['status'] != 'PRESENT': - structure = True - if not sub.get('scan_results', None): - runtime = True - else: - rcodes = [r['scan_results']['returncode'] for r in sub['scan_results'] if r.get('scan_results', None)] - if sum(rcodes) > 0: + + if not args.remove: + filename_complete = f"{args.project_name}-{args.version_name}-{timestamp}-full.json" + filename_failures = f"{args.project_name}-{args.version_name}-{timestamp}-failures.json" + # write full processing log + with open (filename_complete, "w") as f: + json.dump(mipm.project_data, f, indent=2) + + failures = list() + for sname, sub in mipm.project_data['subprojects'].items(): + structure = False + runtime = False + if sub['status'] != 'PRESENT': + structure = True + if not sub.get('scan_results', None): runtime = True - if structure or runtime: - failures.append(sub) - with open (filename_failures, "w") as f: - json.dump(failures, f, indent=2) + else: + rcodes = [r['scan_results']['returncode'] for r in sub['scan_results'] if r.get('scan_results', None)] + if sum(rcodes) > 0: + runtime = True + if structure or runtime: + failures.append(sub) + with open (filename_failures, "w") as f: + json.dump(failures, f, indent=2) if __name__ == "__main__": sys.exit(main()) From 7d32fb26cfefb239c0bf784a134e65dfb75c55be Mon Sep 17 00:00:00 2001 From: "smiths@synopsys.com" <120398516+snps-steve@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:15:23 -0500 Subject: [PATCH 114/146] Add files via upload This version adds "File Paths", "How to Fix" (aka solution), and "References and Related Links". --- ...sv_reports_for_project_version_enhanced.py | 246 ++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 examples/generate_csv_reports_for_project_version_enhanced.py diff --git a/examples/generate_csv_reports_for_project_version_enhanced.py b/examples/generate_csv_reports_for_project_version_enhanced.py new file mode 100644 index 00000000..86e18493 --- /dev/null +++ b/examples/generate_csv_reports_for_project_version_enhanced.py @@ -0,0 +1,246 @@ +''' +Created on Dec 19, 2018 +Updated on Sept 20, 2024 + +@author: gsnyder +@contributor: smiths + +Generate a CSV report for a given project-version and enhance with "File Paths", "How to Fix", and +"References and Related Links" +''' + +import argparse +import csv +import io +import json +import logging +import time +import zipfile +from blackduck.HubRestApi import HubInstance +from requests.exceptions import MissingSchema + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +version_name_map = { + 'version': 'VERSION', + 'scans': 'CODE_LOCATIONS', + 'components': 'COMPONENTS', + 'vulnerabilities': 'SECURITY', + 'source': 'FILES', + 'cryptography': 'CRYPTO_ALGORITHMS', + 'license_terms': 'LICENSE_TERM_FULFILLMENT', + 'component_additional_fields': 'BOM_COMPONENT_CUSTOM_FIELDS', + 'project_version_additional_fields': 'PROJECT_VERSION_CUSTOM_FIELDS', + 'vulnerability_matches': 'VULNERABILITY_MATCH' +} + +all_reports = list(version_name_map.keys()) + +parser = argparse.ArgumentParser("A program to create reports for a given project-version") +parser.add_argument("project_name") +parser.add_argument("version_name") +parser.add_argument("-z", "--zip_file_name", default="reports.zip") +parser.add_argument("-r", "--reports", + default=",".join(all_reports), + help=f"Comma separated list (no spaces) of the reports to generate - {list(version_name_map.keys())}. Default is all reports.", + type=lambda s: s.upper()) +parser.add_argument('--format', default='CSV', choices=["CSV"], help="Report format - only CSV available for now") +parser.add_argument('-t', '--tries', default=5, type=int, help="How many times to retry downloading the report, i.e. wait for the report to be generated") +parser.add_argument('-s', '--sleep_time', default=30, type=int, help="The amount of time to sleep in-between (re-)tries to download the report") + +args = parser.parse_args() + +hub = HubInstance() + +class FailedReportDownload(Exception): + pass + +def download_report(location, filename, retries=args.tries): + report_id = location.split("/")[-1] + + for attempt in range(retries): + + # Wait for 30 seconds before attempting to download + print(f"Waiting 30 seconds before attempting to download...") + time.sleep(30) + + # Retries + print(f"Attempt {attempt + 1} of {retries} to retrieve report {report_id}") + + # Report Retrieval + print(f"Retrieving generated report from {location}") + response = hub.download_report(report_id) + + if response.status_code == 200: + with open(filename, "wb") as f: + f.write(response.content) + print(f"Successfully downloaded zip file to {filename} for report {report_id}") + return response.content + else: + print(f"Failed to retrieve report {report_id}") + if attempt < retries - 1: # If it's not the last attempt + wait_time = args.sleep_time + print(f"Waiting {wait_time} seconds before retrying...") + time.sleep(wait_time) + else: + print(f"Maximum retries reached. Unable to download report.") + + raise FailedReportDownload(f"Failed to retrieve report {report_id} after {retries} tries") + +def get_file_paths(hub, project_id, project_version_id, component_id, component_version_id, component_origin_id): + url = f"{hub.get_urlbase()}/api/projects/{project_id}/versions/{project_version_id}/components/{component_id}/versions/{component_version_id}/origins/{component_origin_id}/matched-files" + headers = { + "Accept": "application/vnd.blackducksoftware.bill-of-materials-6+json", + "Authorization": f"Bearer {hub.token}" + } + + logging.debug(f"Making API call to: {url}") + + try: + response = hub.execute_get(url) + if response.status_code == 200: + data = response.json() + file_paths = [] + for item in data.get('items', []): + file_path = item.get('filePath', {}) + composite_path = file_path.get('compositePathContext', '') + if composite_path: + file_paths.append(composite_path) + return file_paths + else: + logging.error(f"Failed to fetch matched files. Status code: {response.status_code}") + return [] + except Exception as e: + logging.error(f"Error making API request: {str(e)}") + return [] + +def get_vulnerability_details(hub, vulnerability_id): + url = f"{hub.get_urlbase()}/api/vulnerabilities/{vulnerability_id}" + + try: + response = hub.execute_get(url) + if response.status_code == 200: + data = response.json() + solution = data.get('solution', '') + references = [] + meta_data = data.get('_meta', {}) + links = meta_data.get('links', []) + for link in links: + references.append({ + 'rel': link.get('rel', ''), + 'href': link.get('href', '') + }) + return solution, references + else: + logging.error(f"Failed to fetch vulnerability details. Status code: {response.status_code}") + return '', [] + except Exception as e: + logging.error(f"Error making API request for vulnerability details: {str(e)}") + return '', [] + +def enhance_security_report(hub, zip_content, project_id, project_version_id): + logging.info(f"Enhancing security report for Project ID: {project_id}, Project Version ID: {project_version_id}") + + with zipfile.ZipFile(io.BytesIO(zip_content), 'r') as zin: + csv_files = [f for f in zin.namelist() if f.endswith('.csv')] + for csv_file in csv_files: + csv_content = zin.read(csv_file).decode('utf-8') + reader = csv.DictReader(io.StringIO(csv_content)) + + # Count total rows + total_rows = sum(1 for row in reader) + reader = csv.DictReader(io.StringIO(csv_content)) # Reset reader + + fieldnames = reader.fieldnames + ["File Paths", "How to Fix", "References and Related Links"] + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + + processed_components = 0 + skipped_components = 0 + + for index, row in enumerate(reader, 1): + print(f"\rProcessing row {index} of {total_rows} ({index/total_rows*100:.2f}%)", end='', flush=True) + + component_id = row.get('Component id', '') + component_version_id = row.get('Version id', '') + component_origin_id = row.get('Origin id', '') + vulnerability_id = row.get('Vulnerability id', '') + + if not all([component_id, component_version_id, component_origin_id]): + logging.warning(f"Missing component information. Component ID: {component_id}, Component Version ID: {component_version_id}, Origin ID: {component_origin_id}") + skipped_components += 1 + file_paths = [] + else: + file_paths = get_file_paths(hub, project_id, project_version_id, component_id, component_version_id, component_origin_id) + processed_components += 1 + + if vulnerability_id: + solution, references = get_vulnerability_details(hub, vulnerability_id) + else: + solution, references = '', [] + + row["File Paths"] = '; '.join(file_paths) if file_paths else "No file paths available" + row["How to Fix"] = solution + row["References and Related Links"] = json.dumps(references) + + writer.writerow(row) + + print("\nProcessing complete.") + + # Generate a unique filename for the enhanced report + timestamp = time.strftime("%Y%m%d-%H%M%S") + enhanced_filename = f"enhanced_security_report_{timestamp}.csv" + + # Update zip file with modified CSV + with zipfile.ZipFile(args.zip_file_name, 'a') as zout: + zout.writestr(enhanced_filename, output.getvalue()) + + logging.info(f"Enhanced security report saved to {args.zip_file_name}") + logging.info(f"Processed components: {processed_components}") + logging.info(f"Skipped components: {skipped_components}") + +def main(): + hub = HubInstance() + + project = hub.get_project_by_name(args.project_name) + + if project: + project_id = project['_meta']['href'].split('/')[-1] + logging.info(f"Project ID: {project_id}") + + version = hub.get_version_by_name(project, args.version_name) + if version: + project_version_id = version['_meta']['href'].split('/')[-1] + logging.info(f"Project Version ID: {project_version_id}") + + reports_l = [version_name_map.get(r.strip().lower(), r.strip()) for r in args.reports.split(",")] + + valid_reports = set(version_name_map.values()) + invalid_reports = [r for r in reports_l if r not in valid_reports] + if invalid_reports: + print(f"Error: Invalid report type(s): {', '.join(invalid_reports)}") + print(f"Valid report types are: {', '.join(valid_reports)}") + exit(1) + + response = hub.create_version_reports(version, reports_l, args.format) + + if response.status_code == 201: + print(f"Successfully created reports ({args.reports}) for project {args.project_name} and version {args.version_name}") + location = response.headers['Location'] + zip_content = download_report(location, args.zip_file_name) + + if 'SECURITY' in reports_l: + enhance_security_report(hub, zip_content, project_id, project_version_id) + else: + print(f"Failed to create reports for project {args.project_name} version {args.version_name}, status code returned {response.status_code}") + else: + print(f"Did not find version {args.version_name} for project {args.project_name}") + else: + print(f"Did not find project with name {args.project_name}") + +if __name__ == "__main__": + main() \ No newline at end of file From 2f8b9b0002dca68202bd37e3c413c09863acc9dc Mon Sep 17 00:00:00 2001 From: Dinesh Ravi Date: Mon, 23 Sep 2024 19:38:59 +0200 Subject: [PATCH 115/146] Create get_scan_missed_import_event.py Gather list of non matched components where blackduck could not able have match event with their kb for the bdio codelocation type --- .../client/get_scan_missed_import_event.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 examples/client/get_scan_missed_import_event.py diff --git a/examples/client/get_scan_missed_import_event.py b/examples/client/get_scan_missed_import_event.py new file mode 100644 index 00000000..6266e240 --- /dev/null +++ b/examples/client/get_scan_missed_import_event.py @@ -0,0 +1,96 @@ +""" +Created on july 11, 2024 +@author: Dinesh Ravi +Gather list of non matched components where blackduck could not able have match event with their kb for the bdio codelocation type +""" +from blackduck import Client + +import argparse + +import logging +import json +# py get_scan_missed_import_event.py --base-url=https://blackduck.company.com --token-file=.pt --project=ASTERIX2CLU3D_PR +# OGRAM --version=AED2_ANDROID_S_2_2024-09-21_00-32 --company=company --no-verify > missing.txt +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser("Get the BOM components for a given project-version and the license details for each BOM component") +parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token") +parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") +parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") +parser.add_argument("--company", dest='company_name', required=True, help="modules that contains the company name for separation") +parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification") +args = parser.parse_args() + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) + +params = { + 'q': [f"name:{args.project_name}"] +} + +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +params = { + 'q': [f"versionName:{args.version_name}"] +} +versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name] +assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}" +version = versions[0] + +print(f"Found {project['name']}:{version['versionName']}") +hashset_company = set() +hashset_other = set() +# =================== +# print(version) +# params = { +# 'q': [f"name:*bdio*"] +# } +codelocations=[codelocation for codelocation in bd.get_resource('codelocations',version) if 'bdio' in codelocation['name']] +# print(codelocations[0]) +# codelocation=codelocations[0] +for codelocation in codelocations: + scans=bd.get_resource('scans',codelocation) + for scan in scans: + events=[events for events in bd.get_resource('component-import-events',scan) if events['event']=="COMPONENT_MAPPING_FAILED"] + if len(events)>0: + print("============================") + print(f"codelocation_name: {codelocation['name']}") + print(f"matchCount: {scan['matchCount']}") + print(f"missing: {len(events)}") + + for i,event in enumerate(events,start=1): + print(f"--------------{i}") + externalId=event['externalId'] + print(f"externalId: {externalId}") + print(f"importComponentName: {event['importComponentName']}") + print(f"importComponentVersionName: {event['importComponentVersionName']}") + if args.company_name in externalId: + hashset_company.add(externalId) + else: + hashset_other.add(externalId) + + # {'event': 'COMPONENT_MAPPING_FAILED', + # 'importComponentName': 'rsi-common-lib', + # 'importComponentVersionName': '0.2.14', + # 'externalId': 'com.company.aed2:rsi-common-lib:0.2.14', + # 'failureReason': 'Unable to map scanned component version to Black Duck project version because no mapping is present for the given external identifier'} +print("============================") +sorted_company=sorted(hashset_company) +sorted_other=sorted(hashset_other) + +print(f"========total missing Other components to get foss report: {len(hashset_other)}========") +for i,missing in enumerate(sorted_other,start=1): + print(f"{i} {missing}") +print(f"========total missing {args.company_name} components to manually add: {len(hashset_company)}========") +for i,missing in enumerate(sorted_company,start=1): + print(f"{i} {missing}") +total_missing=len(sorted_company)+len(sorted_other) +print(f"========EB:{len(sorted_company)}+Other:{len(sorted_other)}={total_missing}========") From bfd7cfadc30870b7fd96347f134da66f2483c398 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 23 Sep 2024 15:11:38 -0400 Subject: [PATCH 116/146] net add --- examples/client/net_add_components.py | 96 +++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 examples/client/net_add_components.py diff --git a/examples/client/net_add_components.py b/examples/client/net_add_components.py new file mode 100644 index 00000000..d8484d7f --- /dev/null +++ b/examples/client/net_add_components.py @@ -0,0 +1,96 @@ +#$!/usr/bin/env python3 +# + +''' +Created on Sep 23, 2024 +@author: kumykov + +Net Add Components. + +This script will extract components form a source project and add them as +Manually Added components to the target project + +''' +from blackduck import Client + +import argparse +import logging +import sys +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + assert len(projects) == 1, f"Project {project_name} not found." + return projects[0] + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + assert len(versions) == 1, f"Project version {version_name} for project {project['name']} not found" + return versions[0] + +def parse_command_args(): + + parser = argparse.ArgumentParser("Extract components form a source project and add them to the target project as manual components.\n") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument('-sp', '--source-project', help="Source Project") + parser.add_argument('-sv', '--source-version', help="Source Project Version") + parser.add_argument('-tp', '--target-project', help="Target Project") + parser.add_argument('-tv', '--target-version', help="Target Project Version") + + return parser.parse_args() + +def add_component_to_a_project_version(bd, component, components_url): + headers = { + "Content-Type": "application/vnd.blackducksoftware.bill-of-materials-6+json" + } + data = dict() + component_version = component.get('componentVersion', component.get('component')) + component_origins = component['origins'] + if len(component_origins) > 0: + for origin in component_origins: + origin_url = origin['origin'] + payload = {"component": origin_url} + result = bd.session.post(components_url, json=payload, headers=headers) + pprint(result) + else: + data['component'] = component_version + result = bd.session.post(components_url, json=data, headers=headers) + pprint(result) + + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + global bd + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + + project = find_project_by_name(bd, args.source_project) + version = find_project_version_by_name(bd, project, args.source_version) + + target_project = find_project_by_name(bd, args.target_project) + target_version = find_project_version_by_name(bd, target_project, args.target_version) + + dict = bd.list_resources(target_version) + components_url = dict['components'] + + components = bd.get_resource('components', version) + for component in components: + add_component_to_a_project_version(bd, component, components_url) + + +if __name__ == "__main__": + sys.exit(main()) From 05b055fc8887e96cc2b8e8e2beee242cfdcd5ca3 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 23 Sep 2024 15:20:42 -0400 Subject: [PATCH 117/146] net add --- examples/client/net_add_components.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/client/net_add_components.py b/examples/client/net_add_components.py index d8484d7f..eaa5debe 100644 --- a/examples/client/net_add_components.py +++ b/examples/client/net_add_components.py @@ -59,14 +59,17 @@ def add_component_to_a_project_version(bd, component, components_url): data = dict() component_version = component.get('componentVersion', component.get('component')) component_origins = component['origins'] + component_license = component['licenses'][0]['license'] if len(component_origins) > 0: for origin in component_origins: origin_url = origin['origin'] payload = {"component": origin_url} + payload['license'] = component_license result = bd.session.post(components_url, json=payload, headers=headers) pprint(result) else: data['component'] = component_version + data['license'] = component_license result = bd.session.post(components_url, json=data, headers=headers) pprint(result) From 24d5c9bd0dd0225c7d9ca88e98cb53d905576f3a Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 2 Oct 2024 00:11:14 -0400 Subject: [PATCH 118/146] reporcessing option added --- .../multi-image/manage_project_structure.py | 112 ++++++++++++++---- .../multi-image/scan_docker_image_lite.py | 6 +- 2 files changed, 92 insertions(+), 26 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index e29de739..de95b297 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -61,6 +61,8 @@ import sys import arrow +from io import StringIO + from blackduck import Client from pprint import pprint,pformat @@ -75,12 +77,48 @@ def __init__(self, args): with open(args.token_file, 'r') as tf: self.access_token = tf.readline().strip() self.no_verify = args.no_verify + self.reprocess_run_file = args.reprocess_run_file self.connect() - self.init_project_data(args) + if self.reprocess_run_file: + self.load_project_data() + else: + self.init_project_data(args) def connect(self): self.client = Client(base_url=self.base_url, token=self.access_token, verify=self.no_verify, timeout=60.0, retries=4) - + + def load_project_data(self): + with open(self.reprocess_run_file, "r") as f: + data = json.load(f) + self.project_data = data + self.project_data.pop('log', None) + discard = list() + for name, subproject in self.project_data['subprojects'].items(): + if not self.has_errors(subproject): + discard.append(name) + else: + self.project_data['subprojects'][name].pop('log', None) + self.project_data['subprojects'][name].pop('status', None) + self.project_data['subprojects'][name].pop('scan_results',None) + for name in discard: + del self.project_data['subprojects'][name] + + def has_errors(self, subproject): + structure = False + runtime = False + if subproject['status'] != 'PRESENT': + structure = True + if not subproject.get('scan_results', None): + runtime = True + else: + rcodes = [r['scan_results']['returncode'] for r in subproject['scan_results'] if r.get('scan_results', None)] + if sum(rcodes) > 0: + runtime = True + if structure or runtime: + return True + else: + return False + def init_project_data(self,args): self.project_data = dict() self.project_data['project_name'] = args.project_name @@ -435,6 +473,42 @@ def proceed(self): self.validate_project_structure() self.scan_container_images() +def write_failure_report(data, output_file_name): + s = StringIO() + subprojects = data['subprojects'] + for subproject_name, subproject in subprojects.items(): + structure = False + runtime = False + if subproject['status'] != 'PRESENT': + structure = True + if not subproject.get('scan_results', None): + runtime = True + else: + rcodes = [r['scan_results']['returncode'] for r in subproject['scan_results'] if r.get('scan_results', None)] + if sum(rcodes) > 0: + runtime = True + if structure or runtime: + print (f"\nStatus for {subproject['project_name']} {subproject['version_name']}", file = s) + print (f"\tStructural failures present {structure}", file = s) + print (f"\t Runtime failures present {runtime}\n", file = s) + + if subproject['status'] != 'PRESENT': + for line in subproject['log']: + print ('\t', line, file = s) + scan_results = subproject.get('scan_results',[]) + if len(scan_results) == 0: + print ("No scans were performed", file = s) + else: + for invocation in scan_results: + returncode = invocation['scan_results']['returncode'] + if returncode > 0: + print (f"\n\tScan for {invocation['name']} failed with returncode {returncode}\n", file = s) + stdout = invocation['scan_results']['stdout'].split('\n') + for line in stdout: + if 'ERROR' in line and 'certificates' not in line: + print ('\t', line, file = s) + with open(output_file_name, "w") as f: + f.write(s.getvalue()) def parse_command_args(): @@ -442,8 +516,8 @@ def parse_command_args(): parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") parser.add_argument("-t", "--token-file", required=True, help="File containing access token") parser.add_argument("-pg", "--project_group", required=False, default='Multi-Image', help="Project Group to be used") - parser.add_argument("-p", "--project-name", required=True, help="Project Name") - parser.add_argument("-pv", "--version-name", required=True, help="Project Version Name") + parser.add_argument("-p", "--project-name", required=False, help="Project Name") + parser.add_argument("-pv", "--version-name", required=False, help="Project Version Name") group = parser.add_mutually_exclusive_group() group.add_argument("-sp", "--subproject-list", required=False, help="List of subprojects to generate with subproject:container:tag") group.add_argument("-ssf", "--subproject-spec-file", required=False, help="Excel or txt file containing subproject specification") @@ -456,7 +530,13 @@ def parse_command_args(): parser.add_argument("--strict", action='store_true', help="Fail if existing (sub)project versions already exist") parser.add_argument("--binary", action='store_true', help="Use binary scan for analysis") parser.add_argument("-ifm", "--individual-file-matching", action='store_true', help="Turn Individual file matching on") - return parser.parse_args() + parser.add_argument("--reprocess-run-file", help="Reprocess Failures from previous run report.") + args = parser.parse_args() + if not args.reprocess_run_file and not (args.project_name and args.version_name): + parser.error("[ -p/--project-name and -pv/--version-name ] or --reprocess-run-file are required") + if args.reprocess_run_file and (args.project_name or args.version_name): + parser.error("[ -p/--project-name and -pv/--version-name ] or --reprocess-run-file are required") + return args def main(): from datetime import datetime @@ -467,28 +547,14 @@ def main(): mipm.proceed() if not args.remove: - filename_complete = f"{args.project_name}-{args.version_name}-{timestamp}-full.json" - filename_failures = f"{args.project_name}-{args.version_name}-{timestamp}-failures.json" + filename_base = f"{mipm.project_data['project_name']}-{mipm.project_data['version_name']}" + filename_complete = f"{filename_base}-{timestamp}-full.json" + filename_failure_report = f"{filename_base}-{timestamp}-failures.txt" # write full processing log with open (filename_complete, "w") as f: json.dump(mipm.project_data, f, indent=2) - failures = list() - for sname, sub in mipm.project_data['subprojects'].items(): - structure = False - runtime = False - if sub['status'] != 'PRESENT': - structure = True - if not sub.get('scan_results', None): - runtime = True - else: - rcodes = [r['scan_results']['returncode'] for r in sub['scan_results'] if r.get('scan_results', None)] - if sum(rcodes) > 0: - runtime = True - if structure or runtime: - failures.append(sub) - with open (filename_failures, "w") as f: - json.dump(failures, f, indent=2) + write_failure_report(mipm.project_data, filename_failure_report) if __name__ == "__main__": sys.exit(main()) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index e2d912ad..8dd8de9d 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -268,7 +268,7 @@ def prepare_container_image(self): for item in history: if not item.get('empty_layer', None): layer_count += 1 - match = re.search('echo (.+?)_group_end', item['created_by']) + match = re.search('echo (.+?)_group_end', item.get('created_by','')) if match: found = match.group(1) if len(history_grouping): @@ -350,8 +350,8 @@ def process_oci_container_image_by_user_defined_groups(self): self.config = self.docker.read_config() self.layers = self.config['history'] - tagged_layers = [x for x in self.layers if '_group_end' in x.get('created_by')] - groups = {re.search('echo (.+?)_group_end', str(x['created_by'])).group(1): self.layers.index(x) for x in tagged_layers} + tagged_layers = [x for x in self.layers if '_group_end' in x.get('created_by','')] + groups = {re.search('echo (.+?)_group_end', str(x.get('created_by',''))).group(1): self.layers.index(x) for x in tagged_layers} logging.debug(f"Container configuration defines following groups {groups}") layer_paths = self.manifest[0]['Layers'].copy() empty_layers = [x for x in self.layers if x.get('empty_layer', False)] From cf82a624abfc3e503d5e59907f664561621e6fb4 Mon Sep 17 00:00:00 2001 From: Dinesh Ravi Date: Fri, 25 Oct 2024 15:38:58 +0200 Subject: [PATCH 119/146] include_license_info in Reporting.py --- blackduck/Reporting.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/blackduck/Reporting.py b/blackduck/Reporting.py index 70428191..7735a604 100644 --- a/blackduck/Reporting.py +++ b/blackduck/Reporting.py @@ -25,7 +25,7 @@ def create_version_reports(self, version, report_list, format="CSV"): return self.execute_post(version_reports_url, post_data) valid_notices_formats = ["TEXT", "JSON"] -def create_version_notices_report(self, version, format="TEXT", include_copyright_info=True): +def create_version_notices_report(self, version, format="TEXT", include_copyright_info=True, include_license_info=True): assert format in valid_notices_formats, "Format must be one of {}".format(valid_notices_formats) post_data = { @@ -35,6 +35,8 @@ def create_version_notices_report(self, version, format="TEXT", include_copyrigh } if include_copyright_info: post_data.update({'categories': ["COPYRIGHT_TEXT"] }) + if include_license_info: + post_data.update({'categories': ["COPYRIGHT_TEXT","LICENSE_DATA","LICENSE_TEXT"] }) notices_report_url = self.get_link(version, 'licenseReports') return self.execute_post(notices_report_url, post_data) From 2567d47b6fa256544b39b7ccb19babb16d48e59c Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 20 Nov 2024 10:27:26 -0500 Subject: [PATCH 120/146] fixed vuln_info script --- .../client/get_bom_component_vuln_info.py | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/examples/client/get_bom_component_vuln_info.py b/examples/client/get_bom_component_vuln_info.py index 54e4b844..48e01654 100644 --- a/examples/client/get_bom_component_vuln_info.py +++ b/examples/client/get_bom_component_vuln_info.py @@ -49,20 +49,23 @@ all_bom_component_vulns = [] for bom_component_vuln in bd.get_resource('vulnerable-components', version): - vuln_name = bom_component_vuln['vulnerabilityWithRemediation']['vulnerabilityName'] - vuln_source = bom_component_vuln['vulnerabilityWithRemediation']['source'] + vulnerabilities = bd.get_resource('vulnerabilities', bom_component_vuln) upgrade_guidance = bd.get_json(f"{bom_component_vuln['componentVersion']}/upgrade-guidance") bom_component_vuln['upgrade_guidance'] = upgrade_guidance + all_bom_component_vulns.append(bom_component_vuln) + #for vuln in vulnerabilities: + #pprint(vuln) + #vuln_name = vuln['name'] + #vuln_source = vuln['source'] - vuln_details = bd.get_json(f"/api/vulnerabilities/{vuln_name}") - bom_component_vuln['vulnerability_details'] = vuln_details + #vuln_details = bd.get_json(f"/api/vulnerabilities/{vuln_name}") + #bom_component_vuln['vulnerability_details'] = vuln_details - if 'related-vulnerability' in bd.list_resources(vuln_details): - related_vuln = bd.get_resource("related-vulnerability", vuln_details, items=False) - else: - related_vuln = None - bom_component_vuln['related_vulnerability'] = related_vuln - all_bom_component_vulns.append(bom_component_vuln) + #if 'related-vulnerability' in bd.list_resources(vuln_details): + # related_vuln = bd.get_resource("related-vulnerability", vuln_details, items=False) + #else: + # related_vuln = None + #bom_component_vuln['related_vulnerability'] = related_vuln if args.csv_file: '''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints @@ -73,13 +76,13 @@ with open(args.csv_file, 'w') as csv_f: field_names = [ 'Vulnerability Name', - 'Vulnerability Description', + #'Vulnerability Description', 'Remediation Status', 'Component', 'Component Version', - 'Exploit Available', - 'Workaround Available', - 'Solution Available', + #'Exploit Available', + #'Workaround Available', + #'Solution Available', 'Upgrade Guidance - short term', 'Upgrade Guidance - long term', ] @@ -87,14 +90,14 @@ writer.writeheader() for comp_vuln in all_bom_component_vulns: row_data = { - 'Vulnerability Name': comp_vuln['vulnerabilityWithRemediation']['vulnerabilityName'], - 'Vulnerability Description': comp_vuln['vulnerabilityWithRemediation']['description'], - 'Remediation Status': comp_vuln['vulnerabilityWithRemediation']['remediationStatus'], + 'Vulnerability Name': comp_vuln['vulnerability']['vulnerabilityId'], + #'Vulnerability Description': comp_vuln['vulnerabilityWithRemediation']['description'], + 'Remediation Status': comp_vuln['vulnerability']['remediationStatus'], 'Component': comp_vuln['componentName'], 'Component Version': comp_vuln['componentVersionName'], - 'Exploit Available': comp_vuln['vulnerability_details'].get('exploitPublishDate', 'None available'), - 'Workaround Available': comp_vuln['vulnerability_details'].get('workaround', 'None available'), - 'Solution Available': comp_vuln['vulnerability_details'].get('solution', 'None available'), + #'Exploit Available': comp_vuln['vulnerability_details'].get('exploitPublishDate', 'None available'), + #'Workaround Available': comp_vuln['vulnerability_details'].get('workaround', 'None available'), + #'Solution Available': comp_vuln['vulnerability_details'].get('solution', 'None available'), 'Upgrade Guidance - short term': comp_vuln['upgrade_guidance'].get('shortTerm', 'None available'), 'Upgrade Guidance - long term': comp_vuln['upgrade_guidance'].get('longTerm', 'None available') } From dabd48ffda242f662c18c97bf56b68deca62ed41 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Mon, 25 Nov 2024 13:28:48 -0500 Subject: [PATCH 121/146] vuln status report added --- .../client/generate_vuln_status_report.py | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 examples/client/generate_vuln_status_report.py diff --git a/examples/client/generate_vuln_status_report.py b/examples/client/generate_vuln_status_report.py new file mode 100644 index 00000000..cecba564 --- /dev/null +++ b/examples/client/generate_vuln_status_report.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python + +''' +Copyright (C) 2021 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +''' +import argparse +from datetime import datetime +import json +import logging +import requests +import sys +import time +import csv + +from blackduck import Client + +DEFAULT_OUTPUT_FILE="vuln_status_report.csv" + +parser = argparse.ArgumentParser("Generate a vulnerability status report") +parser.add_argument("--base-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", help="containing access token") +parser.add_argument("--projects", nargs="+", help="The list of projects to include in the report") +parser.add_argument('-t', '--tries', default=10, type=int, help="How many times to retry downloading the report, i.e. wait for the report to be generated") +parser.add_argument("-o", "--output-file-name", + dest="file_name", + default=DEFAULT_OUTPUT_FILE, + help=f"Name of the output file (default: {DEFAULT_OUTPUT_FILE})") +parser.add_argument("--no-verify", + dest='verify', + action='store_false', + help="disable TLS certificate verification") +args = parser.parse_args() + + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stdout, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +class FailedReportDownload(Exception): + pass + +def download_vuln_report(bd_client, location, filename, report_format, retries=args.tries): + if retries: + report_status = bd_client.session.get(location).json() + if report_status['status'] == 'COMPLETED': + download_url = bd_client.list_resources(report_status)['download'] + contents_url = download_url + "/contents" + report_contents = bd_client.session.get(contents_url).json() + if report_format == 'JSON': + with open(filename, 'w') as f: + json.dump(report_contents, f, indent=3) + logging.info(f"Wrote vulnerability status report contents to {filename}") + elif report_format == 'CSV': + csv_data = report_contents['reportContent'][0]['fileContent'] + with open(filename, 'w') as f: + f.write(csv_data) + logging.info(f"Wrote vulnerability status report contents to {filename}") + else: + logging.error(f"Unrecognized format ({report_format}) given. Exiting") + + else: + sleep_time = 25 + retries -= 1 + logging.debug(f"Report is not ready to download yet, waiting {sleep_time} seconds and then retrying {retries} more times") + time.sleep(sleep_time) + download_vuln_report(bd_client, location, filename, report_format, retries) + else: + raise FailedReportDownload(f"Failed to retrieve report from {location} after {retries} attempts") + + +def get_projects(client, project_names): + print (project_names) + '''Given a list of project names return a list of the corresponding project URLs''' + project_urls = list() + for project in client.get_items("/api/projects"): + if project['name'] in project_names: + project_urls.append(project['_meta']['href']) + return project_urls + +def augment_filename(filename): + if filename.endswith('.csv'): + index = filename.index('.csv') + return filename[:index] + '_augmented' + filename[index:] + else: + return filename + '_augmented.csv' + +def correct_vuln_ids(bd, filename, new_filename): + logging.debug(f"Generating file with augmented vuln ids as {new_filename}") + input = open(filename, 'r') + reader = csv.DictReader(input) + fieldnames = reader.fieldnames + rowcount = 0 + with open(new_filename, 'w') as output: + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + for row in reader: + vuln_id = row['Vulnerability id'] + related_vuln_id = get_related_vuln_id(bd, vuln_id) + if related_vuln_id: + correct_vuln_id = f"{vuln_id} ({related_vuln_id})" + else: + correct_vuln_id = vuln_id + row['Vulnerability id'] = correct_vuln_id + writer.writerow(row) + rowcount+=1 + if rowcount % 100 == 0: + logging.debug(f"{rowcount:15} rows written into {new_filename}") + logging.debug(f"Total of {rowcount} rows written into {new_filename}") + logging.info(f"Wrote vulnerability status report contents to {filename}") + +def get_related_vuln_id(bd, vuln_id): + related_id = None + if vuln_id.startswith('BDSA') and 'CVE' not in vuln_id: + vuln_data = bd.get_json(f"/api/vulnerabilities/{vuln_id}") + vuln_resources = bd.list_resources(vuln_data) + related_url = vuln_resources.get('related-vulnerability', None) + if related_url: + related_id = related_url.split('/')[-1:][0] + return related_id + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client( + base_url=args.base_url, + token=access_token, + verify=args.verify +) + +project_urls = get_projects(bd, args.projects) +logging.debug(f"Generating vulnerability status report for the following projects: {args.projects}") +logging.debug(f"Project list resulted in following project URLs {project_urls}") +post_data = { + 'reportFormat': 'CSV', + 'projects': project_urls, + 'locale': 'en_US' +} + +try: + r = bd.session.post("/api/vulnerability-status-reports", json=post_data) + r.raise_for_status() + report_url = r.headers['Location'] + logging.debug(f"created vulnerability status report {report_url}") +except requests.HTTPError as err: + # more fine grained error handling here; otherwise: + bd.http_error_handler(err) + logging.error("Failed to generate the report") + sys.exit(1) + +download_vuln_report(bd, report_url, args.file_name, 'CSV', retries=args.tries) + +correct_vuln_ids(bd, args.file_name, augment_filename(args.file_name)) + + + + + + + + + From ae486ab44f9bcfc6195517401a41bbabc0e29e09 Mon Sep 17 00:00:00 2001 From: fanuware Date: Mon, 2 Dec 2024 17:01:39 +0100 Subject: [PATCH 122/146] Fix project creation while SBOM upload --- examples/client/upload_sbom.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/client/upload_sbom.py b/examples/client/upload_sbom.py index 89c1d1de..fd726077 100644 --- a/examples/client/upload_sbom.py +++ b/examples/client/upload_sbom.py @@ -98,7 +98,7 @@ def find_or_create_project_group(group_name): else: return groups[0]['_meta']['href'] -def create_project_version(project_name,version_name,project_group, nickname = None): +def create_project_version(project_name, version_name, project_group, nickname = None): version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name} if nickname: version_data['nickname'] = nickname @@ -120,7 +120,7 @@ def find_or_create_project_version(project_name, version_name, project_group): if version: pass else: - version = create_project_version(project_name, version_name) + version = create_project_version(project_name, version_name, project_group) else: version = create_project_version(project_name, version_name, project_group) project = find_project_by_name(project_name) From c1d6af9a73635e069be8ac3e44abfbad241276f2 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 6 Dec 2024 09:26:41 -0500 Subject: [PATCH 123/146] list projects added --- examples/client/list_projects.py | 104 +++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 examples/client/list_projects.py diff --git a/examples/client/list_projects.py b/examples/client/list_projects.py new file mode 100644 index 00000000..91897f4d --- /dev/null +++ b/examples/client/list_projects.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +''' +Copyright (C) 2023 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +List projects and project owners + +usage: list_projects.py [-h] -u BASE_URL -t TOKEN_FILE [-nv] + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + containing access token + -nv, --no-verify disable TLS certificate verification + +''' +import argparse +import json +import logging +import sys +import arrow +import csv + +from blackduck import Client +from pprint import pprint + +parser = argparse.ArgumentParser("List projects and project owners") +parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", dest='token_file', required=True, help="containing access token") +parser.add_argument("-nv", "--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification") +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client( + base_url=args.base_url, + token=access_token, + verify=args.verify +) + +headers = {} +projects = bd.get_resource('projects') + +def get_user_name(url): + if url: + data = bd.get_json(url) + return (data['userName']) + else: + return None + +fieldnames = ['Project Name', + 'Project Owner', + 'Project Created By', + 'Project Updated By'] + +file_name = 'projects_by_owners.csv' + +with open(file_name, 'w') as output: + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + + for project in projects: + project_name = project['name'] + project_owner = project.get('projectOwner', None) + project_owner_name = get_user_name(project_owner) + project_created_by = project.get('createdByUser', None) + project_created_by_name = get_user_name(project_created_by) + project_updated_by = project.get('updatedByUser', None) + project_updated_by_name = get_user_name(project_updated_by) + project_users = bd.get_resource('users', project) + row = dict() + row['Project Name'] = project_name + row['Project Owner'] = project_owner_name + row['Project Created By'] = project_created_by_name + row['Project Updated By'] = project_updated_by_name + writer.writerow(row) + + logging.info(f"Output file {file_name} written") \ No newline at end of file From c0afd0d8f81efece756a43f43a0f59dfbfac93d0 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 10 Jan 2025 09:23:22 -0500 Subject: [PATCH 124/146] changed tdetect reference to blackduck --- examples/client/batch_generate_sbom.py | 2 +- examples/client/generate_sbom.py | 1 + examples/scan_docker_image_lite.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/client/batch_generate_sbom.py b/examples/client/batch_generate_sbom.py index e27374d1..d8a39a84 100644 --- a/examples/client/batch_generate_sbom.py +++ b/examples/client/batch_generate_sbom.py @@ -262,7 +262,7 @@ def sanitize_filename(filename): def parse_command_args(): - parser = argparse.ArgumentParser("Generate and download reports for projets in a spreadsheet") + parser = argparse.ArgumentParser("Generate and download reports for projects in a spreadsheet") parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") parser.add_argument("-t", "--token-file", required=True, help="File containing access token") parser.add_argument("-i", "--input-file", required=True, help="Project Name") diff --git a/examples/client/generate_sbom.py b/examples/client/generate_sbom.py index 4a0fef00..f2da3249 100644 --- a/examples/client/generate_sbom.py +++ b/examples/client/generate_sbom.py @@ -116,6 +116,7 @@ def download_report(bd_client, location, filename, retries=args.retries): logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") r.raise_for_status() location = r.headers.get('Location') +print(f"location {location}") assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" logging.debug(f"Created SBOM report of type {args.type} for project {args.project_name}, version {args.version_name} at location {location}") diff --git a/examples/scan_docker_image_lite.py b/examples/scan_docker_image_lite.py index 46f1027e..50c988c3 100644 --- a/examples/scan_docker_image_lite.py +++ b/examples/scan_docker_image_lite.py @@ -205,7 +205,7 @@ def __init__(self, hub): # self.detecturl = 'https://detect.synopsys.com/detect.sh' # self.detecturl = 'https://detect.synopsys.com/detect7.sh' # self.detecturl = 'https://detect.synopsys.com/detect8.sh' - self.detecturl = 'https://detect.synopsys.com/detect9.sh' + self.detecturl = 'https://detect.blackduck.com/detect9.sh' self.baseurl = hub.config['baseurl'] self.filename = '/tmp/hub-detect.sh' self.token=hub.config['api_token'] From bb4d9924012be85d7ca028ec0d0ca97f870790d0 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 10 Jan 2025 09:26:07 -0500 Subject: [PATCH 125/146] changed tdetect reference to blackduck --- examples/client/multi-image/scan_docker_image_lite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index 8dd8de9d..9278f898 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -203,7 +203,7 @@ def __init__(self, hub): # self.detecturl = 'https://detect.synopsys.com/detect.sh' # self.detecturl = 'https://detect.synopsys.com/detect7.sh' # self.detecturl = 'https://detect.synopsys.com/detect8.sh' - self.detecturl = 'https://detect.synopsys.com/detect9.sh' + self.detecturl = 'https://detect.blackduck.com/detect9.sh' self.baseurl = hub.config['baseurl'] self.filename = '/tmp/hub-detect.sh' self.token=hub.config['api_token'] From 61f333d76f554904fe0231f26b2a2a1bc6282950 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Fri, 10 Jan 2025 14:55:48 -0500 Subject: [PATCH 126/146] detect 10 line added --- examples/scan_docker_image_lite.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/scan_docker_image_lite.py b/examples/scan_docker_image_lite.py index 50c988c3..723e2075 100644 --- a/examples/scan_docker_image_lite.py +++ b/examples/scan_docker_image_lite.py @@ -206,6 +206,7 @@ def __init__(self, hub): # self.detecturl = 'https://detect.synopsys.com/detect7.sh' # self.detecturl = 'https://detect.synopsys.com/detect8.sh' self.detecturl = 'https://detect.blackduck.com/detect9.sh' + # self.detecturl = 'https://detect.blackduck.com/detect10.sh' self.baseurl = hub.config['baseurl'] self.filename = '/tmp/hub-detect.sh' self.token=hub.config['api_token'] From 855a6e91e97a594cdda850e38f83b1d357546d77 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 15 Jan 2025 13:44:03 -0500 Subject: [PATCH 127/146] added --serialize option for multi-image script --- examples/client/multi-image/manage_project_structure.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index de95b297..55623996 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -83,6 +83,7 @@ def __init__(self, args): self.load_project_data() else: self.init_project_data(args) + self.serialize = args.serialize def connect(self): self.client = Client(base_url=self.base_url, token=self.access_token, verify=self.no_verify, timeout=60.0, retries=4) @@ -438,6 +439,8 @@ def scan_container_images(self): detect_options = (f"--detect.parent.project.name={parent_project} " f"--detect.parent.project.version.name={parent_version} " f"--detect.project.version.nickname={image_name}") + if self.serialize: + detect_options += f" --detect.wait.for.results=true" if self.individual_file_matching: detect_options += f" --detect.blackduck.signature.scanner.individual.file.matching=ALL" if clone_from: @@ -531,6 +534,7 @@ def parse_command_args(): parser.add_argument("--binary", action='store_true', help="Use binary scan for analysis") parser.add_argument("-ifm", "--individual-file-matching", action='store_true', help="Turn Individual file matching on") parser.add_argument("--reprocess-run-file", help="Reprocess Failures from previous run report.") + parser.add_argument("--serialize", action='store_true', help="Serialize scan submissions by adding --detect.wait.for.results=true to scan invocations") args = parser.parse_args() if not args.reprocess_run_file and not (args.project_name and args.version_name): parser.error("[ -p/--project-name and -pv/--version-name ] or --reprocess-run-file are required") From 726e1f7cd6f18026aa774553a726429ef822ae79 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Thu, 16 Jan 2025 13:32:29 -0500 Subject: [PATCH 128/146] execute_patch added to HubInstance per NXP --- blackduck/Core.py | 11 +++++++++++ blackduck/HubRestApi.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/blackduck/Core.py b/blackduck/Core.py index abed0867..53af5eae 100644 --- a/blackduck/Core.py +++ b/blackduck/Core.py @@ -146,6 +146,17 @@ def execute_put(self, url, data, custom_headers={}): response = requests.put(url, headers=headers, data=json_data, verify = not self.config['insecure']) return response +def execute_patch(self, url, data, custom_headers={}) -> requests.Response: + ''' + Work-around to add missing execute_patch() + ''' + json_data = self._validated_json_data(data) + headers = self.get_headers() + headers["Content-Type"] = "application/json" + headers.update(custom_headers) + response = requests.patch(url, headers=headers, data=json_data, verify=not self.config['insecure']) + return response + def _create(self, url, json_body): response = self.execute_post(url, json_body) # v4+ returns the newly created location in the response headers diff --git a/blackduck/HubRestApi.py b/blackduck/HubRestApi.py index d3ddb43e..9fcd0259 100755 --- a/blackduck/HubRestApi.py +++ b/blackduck/HubRestApi.py @@ -68,7 +68,7 @@ class HubInstance(object): _create,_get_hub_rest_api_version_info,_get_major_version,_get_parameter_string,_validated_json_data, execute_delete,execute_get,execute_post,execute_put,get_api_version,get_apibase,get_auth_token,get_headers, get_limit_paramstring,get_link,get_matched_components,get_tags_url,get_urlbase,read_config,write_config, - _check_version_compatibility + _check_version_compatibility,execute_patch ) from .Roles import ( _get_role_url, assign_role_given_role_url, assign_role_to_user_or_group, From 6d817e4da5bf045e05655856abfd7cd8584bb465 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 22 Jan 2025 11:11:28 -0500 Subject: [PATCH 129/146] Updated to detect 10 --- examples/client/multi-image/scan_docker_image_lite.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index 9278f898..8508685f 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -203,7 +203,8 @@ def __init__(self, hub): # self.detecturl = 'https://detect.synopsys.com/detect.sh' # self.detecturl = 'https://detect.synopsys.com/detect7.sh' # self.detecturl = 'https://detect.synopsys.com/detect8.sh' - self.detecturl = 'https://detect.blackduck.com/detect9.sh' + # self.detecturl = 'https://detect.blackduck.com/detect9.sh' + self.detecturl = 'https://detect.blackduck.com/detect10.sh' self.baseurl = hub.config['baseurl'] self.filename = '/tmp/hub-detect.sh' self.token=hub.config['api_token'] From ae82c50ef9606653ba2665580de7d2b66686f9a4 Mon Sep 17 00:00:00 2001 From: mkumykov <38922450+mkumykov@users.noreply.github.com> Date: Wed, 12 Feb 2025 15:58:05 -0500 Subject: [PATCH 130/146] Create README.md --- examples/client/multi-image/README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 examples/client/multi-image/README.md diff --git a/examples/client/multi-image/README.md b/examples/client/multi-image/README.md new file mode 100644 index 00000000..f5c56f1b --- /dev/null +++ b/examples/client/multi-image/README.md @@ -0,0 +1,2 @@ +# Large scal containerized project scan automation +## From 577cfa2c398576ef4fade0eac46446d217c9d588 Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Wed, 12 Feb 2025 16:02:13 -0500 Subject: [PATCH 131/146] spelling --- examples/client/multi-image/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/multi-image/README.md b/examples/client/multi-image/README.md index f5c56f1b..50db637b 100644 --- a/examples/client/multi-image/README.md +++ b/examples/client/multi-image/README.md @@ -1,2 +1,2 @@ -# Large scal containerized project scan automation +# Large scale containerized project scan automation ## From fb80ba07b7084906a0a36e0ac6896dae17933ca6 Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Tue, 18 Feb 2025 15:06:58 +0000 Subject: [PATCH 132/146] Added media type header to API call --- examples/client/get_bom_component_vuln_info.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/examples/client/get_bom_component_vuln_info.py b/examples/client/get_bom_component_vuln_info.py index 48e01654..1da4fb3c 100644 --- a/examples/client/get_bom_component_vuln_info.py +++ b/examples/client/get_bom_component_vuln_info.py @@ -48,7 +48,22 @@ all_bom_component_vulns = [] -for bom_component_vuln in bd.get_resource('vulnerable-components', version): +# version of API to call +api_version = 8 + +media_type = "application/vnd.blackducksoftware.bill-of-materials-" + str(api_version) + "+json" +#media_type = "application/json" + +# lower case keys +lc_keys = {} +lc_keys['accept'] = media_type +lc_keys['content-type'] = media_type + +# keyword arguments to pass +kwargs={} +kwargs['headers'] = lc_keys + +for bom_component_vuln in bd.get_resource('vulnerable-components', version, **kwargs): vulnerabilities = bd.get_resource('vulnerabilities', bom_component_vuln) upgrade_guidance = bd.get_json(f"{bom_component_vuln['componentVersion']}/upgrade-guidance") bom_component_vuln['upgrade_guidance'] = upgrade_guidance From 53dc6662b92f7c33c323fed0a619294a222ab3cd Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 18 Feb 2025 13:55:34 -0500 Subject: [PATCH 133/146] suppress project data parsing when removing project --- examples/client/multi-image/manage_project_structure.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 55623996..7a89f26f 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -129,6 +129,8 @@ def init_project_data(self,args): self.project_data['dry_run'] = args.dry_run self.project_data['strict'] = args.strict self.project_data['remove'] = args.remove + if args.remove: + return subprojects = dict() child_spec_list = self.get_child_spec_list(args) @@ -547,7 +549,8 @@ def main(): timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") args = parse_command_args() mipm = MultiImageProjectManager(args) - logging.info(f"Parsed {len(mipm.project_data['subprojects'])} projects from specification data") + if not args.remove: + logging.info(f"Parsed {len(mipm.project_data['subprojects'])} projects from specification data") mipm.proceed() if not args.remove: From 47cab2e8bfbfe3f8a817ba1d7a15b1d243dd239b Mon Sep 17 00:00:00 2001 From: Murat Kumykov Date: Tue, 18 Feb 2025 17:00:56 -0500 Subject: [PATCH 134/146] . --- examples/client/multi-image/manage_project_structure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index 7a89f26f..c648e63c 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -320,8 +320,8 @@ def process_text_spec_file(self,args): image_name = line.split('/')[-1].split(':')[0] # Don't look at me, you wrote it! sub_project_name = "_".join((prefix, image_name)) spec_line = ":".join((sub_project_name, line)) - if "ciena.com" in spec_line: - project_list.append(spec_line) + # if "ciena.com" in spec_line: + project_list.append(spec_line) return (project_list) def get_child_spec_list(self,args): From bce5e7057e357c7619f959ef61687e9570dc735b Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Thu, 22 May 2025 14:31:17 +0100 Subject: [PATCH 135/146] Create refresh_project_copyrights.py New example to bulk refresh copyrights in components --- examples/client/refresh_project_copyrights.py | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 examples/client/refresh_project_copyrights.py diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py new file mode 100644 index 00000000..d09fca5b --- /dev/null +++ b/examples/client/refresh_project_copyrights.py @@ -0,0 +1,253 @@ +# Iterate through components within a named project (or all) and named version (or all) +# and refresh the copyrights of each - equivalent to clicking the UI refresh button +# Use --debug for some feedback on progress +# Ian Ashworth, May 2025 +# +import http.client +from sys import api_version +import csv +import datetime +from blackduck import Client +import argparse +import logging +from pprint import pprint +import array as arr + +http.client._MAXHEADERS = 1000 + +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +def RepDebug(level, msg): + if hasattr(args, 'debug') and level <= args.debug: + print("dbg{" + str(level) + "} " + msg) + return True + return False + + +# Parse command line arguments +parser = argparse.ArgumentParser("Refresh copyrights for project/version components") + +parser.add_argument("--base-url", required=True, help="BD Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", dest='token_file', required=True, help="File containing your access token") + +parser.add_argument("--dump-data", dest='dump_data', action='store_true', help="Retain analysed data") +parser.add_argument("--csv-file", dest='csv_file', help="File name for dumped data formatted as CSV") + +parser.add_argument("--project", dest='project_name', help="Project name") +parser.add_argument("--version", dest='version_name', help="Version name") + +parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum projects to inspect else all") +parser.add_argument("--max-versions-per-project", dest='max_versions_per_project', type=int, help="Maximum versions per project to inspect else all") +parser.add_argument("--max-components", dest='max_components', type=int, help="Maximum components to inspect in total else all") + +parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none)") + +parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") +args = parser.parse_args() + +# open the access token file +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +# access the Black Duck platform +bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) + +# initialise +all_my_comp_data = [] +my_statistics = {} + + +# version of components API to call +comp_api_version = 6 + +comp_accept_version = "application/vnd.blackducksoftware.bill-of-materials-" + str(comp_api_version) + "+json" +#comp_accept_version = "application/json" + +comp_content_type = comp_accept_version + +# header keys +comp_lc_keys = {} +comp_lc_keys['accept'] = comp_accept_version +comp_lc_keys['content-type'] = comp_accept_version + +# keyword arguments to pass to API call +comp_kwargs={} +comp_kwargs['headers'] = comp_lc_keys + + +# version of API to call +refresh_api_version = 4 + +refresh_accept_version = "application/vnd.blackducksoftware.copyright-" + str(refresh_api_version) + "+json" +#refresh_accept_version = "application/json" + +refresh_content_type = refresh_accept_version + + +# header keys +refresh_lc_keys = {} +refresh_lc_keys['accept'] = refresh_accept_version +refresh_lc_keys['content-type'] = refresh_accept_version + +# keyword arguments to pass to API call +refresh_kwargs={} +refresh_kwargs['headers'] = refresh_lc_keys + + +# zero our main counters +my_statistics['_cntProjects'] = 0 +my_statistics['_cntVersions'] = 0 +my_statistics['_cntComponents'] = 0 +my_statistics['_cntRefresh'] = 0 + +# record any control values +if args.project_name: + my_statistics['_namedProject'] = args.project_name +if args.version_name: + my_statistics['_namedVersion'] = args.version_name + +if args.max_projects: + my_statistics['_maxProjects'] = args.max_projects +if args.max_versions_per_project: + my_statistics['_maxVersionsPerProject'] = args.max_versions_per_project +if args.max_components: + my_statistics['_maxComponents'] = args.max_components + +now = datetime.datetime.now() +print('Started: %s' % now.strftime("%Y-%m-%d %H:%M:%S")) + +# check named project of specific interest +if args.project_name: + params = { + 'q': [f"name:{args.project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] + + # must exist + assert len(projects) > 0, f"There should be at least one - {len(projects)} project(s) noted" +else: + # all projects are in scope + projects = bd.get_resource('projects') + +# loop through projects list +for this_project in projects: + + # check if we have hit any limit + if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + break + + if args.max_projects and my_statistics['_cntProjects'] >= args.max_projects: + break + + my_statistics['_cntProjects'] += 1 + RepDebug(1, '## Project %d: %s' % (my_statistics['_cntProjects'], this_project['name'])) + + if args.version_name: + # note the specific project version of interest + params = { + 'q': [f"versionName:{args.version_name}"] + } + versions = [v for v in bd.get_resource('versions', this_project, params=params) if v['versionName'] == args.version_name] + + # it must exist + assert len(versions) > 0, f"There should be at least one - {len(versions)} version(s) noted" + else: + # all versions for this project are in scope + versions = bd.get_resource('versions', this_project) + + nVersionsPerProject = 0 + + for this_version in versions: + + # check if we have hit any limit + if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + # exit component loop - at the limit + break + + if args.max_versions_per_project and nVersionsPerProject >= args.max_versions_per_project: + # exit loop - at the version per project limit + break + + nVersionsPerProject += 1 + my_statistics['_cntVersions'] += 1 + + # Announce +# logging.debug(f"Found {this_project['name']}:{this_version['versionName']}") + RepDebug(3, ' Version: %s' % this_version['versionName']) + + + # iterate through all components for this project version + for this_comp_data in bd.get_resource('components', this_version, **comp_kwargs): + + if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + # exit component loop - at the limit + break + + my_statistics['_cntComponents'] += 1 + RepDebug(4, ' Component: %s' % this_comp_data['componentName']) + + # refresh the copyrights for this component + url = this_comp_data['origins'][0]['origin'] + url += "/copyrights-refresh" + + response = bd.session.put(url, data=None, **refresh_kwargs) + RepDebug(5,'Refresh response %s' % response) + + inputExternalIds = this_comp_data['inputExternalIds'][0] + RepDebug(2, ' ID: %s' % inputExternalIds) + + my_statistics['_cntRefresh'] += 1 + + # if recording the data - perhaps outputting to a CSV file + if args.dump_data: + my_data = {} + my_data['componentName'] = this_comp_data['componentName'] + my_data['componentVersion'] = this_comp_data['componentVersionName'] + + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + + # add to our list + all_my_comp_data.append(my_data) + + +# end of processing loop + +now = datetime.datetime.now() +print('Finished: %s' % now.strftime("%Y-%m-%d %H:%M:%S")) +print('Summary:') +pprint(my_statistics) + +# if dumping data +if args.dump_data: + # if outputting to a CSV file + if args.csv_file: + '''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints + for a complete list of the fields available. The below code shows a subset of them just to + illustrate how to write out the data into a CSV format. + ''' + logging.info(f"Exporting {len(all_my_comp_data)} records to CSV file {args.csv_file}") + + with open(args.csv_file, 'w') as csv_f: + field_names = [ + 'Component', + 'Component Version' + ] + + writer = csv.DictWriter(csv_f, fieldnames=field_names) + writer.writeheader() + + for my_comp_data in all_my_comp_data: + row_data = { + 'Component': my_comp_data['componentName'], + 'Component Version': my_comp_data['componentVersion'] + } + writer.writerow(row_data) + else: + # print to screen + pprint(all_my_comp_data) + +#end From 5ed3923c5ee88f5984cec3c7013deecbd41e1c72 Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Thu, 22 May 2025 16:15:58 +0100 Subject: [PATCH 136/146] Update refresh_project_copyrights.py Added error handling when no origin and no external ID --- examples/client/refresh_project_copyrights.py | 68 +++++++++++++++---- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py index d09fca5b..366e8489 100644 --- a/examples/client/refresh_project_copyrights.py +++ b/examples/client/refresh_project_copyrights.py @@ -5,6 +5,7 @@ # import http.client from sys import api_version +import sys import csv import datetime from blackduck import Client @@ -26,6 +27,10 @@ def RepDebug(level, msg): return True return False +def RepWarning(msg): + print("WARNING: " + msg) + return True + # Parse command line arguments parser = argparse.ArgumentParser("Refresh copyrights for project/version components") @@ -46,6 +51,9 @@ def RepDebug(level, msg): parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none)") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") +parser.add_argument("-t", "--timeout", default=15, type=int, help="Adjust the (HTTP) session timeout value (default: 15s)") +parser.add_argument("-r", "--retries", default=3, type=int, help="Adjust the number of retries on failure (default: 3)") + args = parser.parse_args() # open the access token file @@ -53,7 +61,13 @@ def RepDebug(level, msg): access_token = tf.readline().strip() # access the Black Duck platform -bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) +bd = Client( + base_url=args.base_url, + token=access_token, + verify=args.verify, + timeout=args.timeout, + retries=args.retries, +) # initialise all_my_comp_data = [] @@ -102,6 +116,9 @@ def RepDebug(level, msg): my_statistics['_cntVersions'] = 0 my_statistics['_cntComponents'] = 0 my_statistics['_cntRefresh'] = 0 +my_statistics['_cntNoOrigins'] = 0 +my_statistics['_cntNoIDs'] = 0 + # record any control values if args.project_name: @@ -187,25 +204,48 @@ def RepDebug(level, msg): break my_statistics['_cntComponents'] += 1 - RepDebug(4, ' Component: %s' % this_comp_data['componentName']) - - # refresh the copyrights for this component - url = this_comp_data['origins'][0]['origin'] - url += "/copyrights-refresh" + RepDebug(4, ' Component: %s (%s)' % + (this_comp_data['componentName'], this_comp_data['componentVersionName'])) + + if this_comp_data['inputExternalIds'].__len__() > 0: + inputExternalIds = this_comp_data['inputExternalIds'][0] + else: + my_statistics['_cntNoIDs'] += 1 + inputExternalIds = "n/a" + RepDebug(2, ' ID: %s' % inputExternalIds) - response = bd.session.put(url, data=None, **refresh_kwargs) - RepDebug(5,'Refresh response %s' % response) - inputExternalIds = this_comp_data['inputExternalIds'][0] - RepDebug(2, ' ID: %s' % inputExternalIds) + # refresh the copyrights for this component + if this_comp_data['origins'].__len__() > 0: + url = this_comp_data['origins'][0]['origin'] + else: + # no origins + RepWarning('No origin defined for [%s]' % this_comp_data['componentVersion']) +# url = this_comp_data['componentVersion'] + url = '' + + if len(url) > 0: + # refresh end point + url += "/copyrights-refresh" + + try: + response = bd.session.put(url, data=None, **refresh_kwargs) + RepDebug(5,'Refresh response %s' % response) + except urllib3.exceptions.ReadTimeoutError: + print('Failed to confirm copyrights refresh') + + my_statistics['_cntRefresh'] += 1 + else: + my_statistics['_cntNoOrigins'] += 1 + url = 'n/a' - my_statistics['_cntRefresh'] += 1 # if recording the data - perhaps outputting to a CSV file if args.dump_data: my_data = {} my_data['componentName'] = this_comp_data['componentName'] my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['url'] = url if hasattr(args, 'debug') and 5 <= args.debug: pprint(my_data) @@ -234,7 +274,8 @@ def RepDebug(level, msg): with open(args.csv_file, 'w') as csv_f: field_names = [ 'Component', - 'Component Version' + 'Component Version', + 'Url' ] writer = csv.DictWriter(csv_f, fieldnames=field_names) @@ -243,7 +284,8 @@ def RepDebug(level, msg): for my_comp_data in all_my_comp_data: row_data = { 'Component': my_comp_data['componentName'], - 'Component Version': my_comp_data['componentVersion'] + 'Component Version': my_comp_data['componentVersion'], + 'Url': my_comp_data['url'] } writer.writerow(row_data) else: From 053a6473f7f50a45ce5bdf8e3bf03dd61ceae6e5 Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Fri, 23 May 2025 09:29:07 +0100 Subject: [PATCH 137/146] Update refresh_project_copyrights.py Added dry run option --- examples/client/refresh_project_copyrights.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py index 366e8489..fbbcf2a9 100644 --- a/examples/client/refresh_project_copyrights.py +++ b/examples/client/refresh_project_copyrights.py @@ -48,7 +48,8 @@ def RepWarning(msg): parser.add_argument("--max-versions-per-project", dest='max_versions_per_project', type=int, help="Maximum versions per project to inspect else all") parser.add_argument("--max-components", dest='max_components', type=int, help="Maximum components to inspect in total else all") -parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none)") +parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none 'n'=level)") +parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") parser.add_argument("-t", "--timeout", default=15, type=int, help="Adjust the (HTTP) session timeout value (default: 15s)") @@ -228,13 +229,17 @@ def RepWarning(msg): # refresh end point url += "/copyrights-refresh" - try: - response = bd.session.put(url, data=None, **refresh_kwargs) - RepDebug(5,'Refresh response %s' % response) - except urllib3.exceptions.ReadTimeoutError: - print('Failed to confirm copyrights refresh') + if args.dry_run != 0: + RepDebug(1, "DryRun: %s" % url) + else: + try: + response = bd.session.put(url, data=None, **refresh_kwargs) + RepDebug(5,'Refresh response %s' % response) + except ReadTimeoutError: + print('Failed to confirm copyrights refresh') + + my_statistics['_cntRefresh'] += 1 - my_statistics['_cntRefresh'] += 1 else: my_statistics['_cntNoOrigins'] += 1 url = 'n/a' From aea5f58173289a4d9af4e155e9b7b0287a1622ea Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Fri, 23 May 2025 14:35:05 +0100 Subject: [PATCH 138/146] Update refresh_project_copyrights.py Iterate through each origin per component and refresh each, increase timeout to 60 seconds, added dry run option --- examples/client/refresh_project_copyrights.py | 104 +++++++++++------- 1 file changed, 66 insertions(+), 38 deletions(-) diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py index fbbcf2a9..4244b4b5 100644 --- a/examples/client/refresh_project_copyrights.py +++ b/examples/client/refresh_project_copyrights.py @@ -14,6 +14,8 @@ from pprint import pprint import array as arr +from urllib3.exceptions import ReadTimeoutError + http.client._MAXHEADERS = 1000 logging.basicConfig( @@ -52,8 +54,8 @@ def RepWarning(msg): parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") -parser.add_argument("-t", "--timeout", default=15, type=int, help="Adjust the (HTTP) session timeout value (default: 15s)") -parser.add_argument("-r", "--retries", default=3, type=int, help="Adjust the number of retries on failure (default: 3)") +parser.add_argument("--timeout", default=60, type=int, help="Adjust the (HTTP) session timeout value (default: 60s)") +parser.add_argument("--retries", default=3, type=int, help="Adjust the number of retries on failure (default: 3)") args = parser.parse_args() @@ -64,8 +66,8 @@ def RepWarning(msg): # access the Black Duck platform bd = Client( base_url=args.base_url, - token=access_token, verify=args.verify, + token=access_token, timeout=args.timeout, retries=args.retries, ) @@ -205,8 +207,9 @@ def RepWarning(msg): break my_statistics['_cntComponents'] += 1 - RepDebug(4, ' Component: %s (%s)' % - (this_comp_data['componentName'], this_comp_data['componentVersionName'])) + comp_label = "{} ({})".format(this_comp_data['componentName'], this_comp_data['componentVersionName']) + + RepDebug(4, ' Component: %s' % comp_label) if this_comp_data['inputExternalIds'].__len__() > 0: inputExternalIds = this_comp_data['inputExternalIds'][0] @@ -218,46 +221,69 @@ def RepWarning(msg): # refresh the copyrights for this component if this_comp_data['origins'].__len__() > 0: - url = this_comp_data['origins'][0]['origin'] - else: - # no origins - RepWarning('No origin defined for [%s]' % this_comp_data['componentVersion']) -# url = this_comp_data['componentVersion'] - url = '' - - if len(url) > 0: - # refresh end point - url += "/copyrights-refresh" - - if args.dry_run != 0: - RepDebug(1, "DryRun: %s" % url) - else: - try: - response = bd.session.put(url, data=None, **refresh_kwargs) - RepDebug(5,'Refresh response %s' % response) - except ReadTimeoutError: - print('Failed to confirm copyrights refresh') - - my_statistics['_cntRefresh'] += 1 + + n_origin = 0 + + for this_origin in this_comp_data['origins']: + + n_origin += 1 + origin_id = this_origin['externalId'] + url = this_origin['origin'] + + # refresh with end point + url += "/copyrights-refresh" + + status = -1 + + if args.dry_run != 0: + RepDebug(1, "DryRun: no=%d origin=%s url=%s" % (n_origin, origin_id, url)) + else: + try: + response = bd.session.put(url, data=None, **refresh_kwargs) + RepDebug(5,'Refresh response: origin [%s] [%s]' % (this_origin, response)) + my_statistics['_cntRefresh'] += 1 + status= 0 + + except Exception: + print('Failed to confirm copyrights refresh') + status = 1 + + + # if recording the data - perhaps outputting to a CSV file + if args.dump_data: + my_data = {} + my_data['componentName'] = this_comp_data['componentName'] + my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['status'] = status + my_data['url'] = url + + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + + # add to our list + all_my_comp_data.append(my_data) else: + # no origins defined + RepWarning('No origin(s) defined for [%s]' % comp_label) my_statistics['_cntNoOrigins'] += 1 + origin_id = '' + status = 3 url = 'n/a' + # if recording the data + if args.dump_data: + my_data = {} + my_data['componentName'] = this_comp_data['componentName'] + my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['status'] = status + my_data['url'] = url - # if recording the data - perhaps outputting to a CSV file - if args.dump_data: - my_data = {} - my_data['componentName'] = this_comp_data['componentName'] - my_data['componentVersion'] = this_comp_data['componentVersionName'] - my_data['url'] = url - - if hasattr(args, 'debug') and 5 <= args.debug: - pprint(my_data) - - # add to our list - all_my_comp_data.append(my_data) + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + # add to our list + all_my_comp_data.append(my_data) # end of processing loop @@ -280,6 +306,7 @@ def RepWarning(msg): field_names = [ 'Component', 'Component Version', + 'Status', 'Url' ] @@ -290,6 +317,7 @@ def RepWarning(msg): row_data = { 'Component': my_comp_data['componentName'], 'Component Version': my_comp_data['componentVersion'], + 'Status': my_comp_data['status'], 'Url': my_comp_data['url'] } writer.writerow(row_data) From ebcd1f4a3650eaae3e12966f7fb45f83abecee21 Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Tue, 27 May 2025 13:11:27 +0100 Subject: [PATCH 139/146] Update refresh_project_copyrights.py Allowances for missing data items like externalId --- examples/client/refresh_project_copyrights.py | 157 +++++++++++++----- 1 file changed, 111 insertions(+), 46 deletions(-) diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py index 366e8489..ed384fe8 100644 --- a/examples/client/refresh_project_copyrights.py +++ b/examples/client/refresh_project_copyrights.py @@ -14,6 +14,8 @@ from pprint import pprint import array as arr +from urllib3.exceptions import ReadTimeoutError + http.client._MAXHEADERS = 1000 logging.basicConfig( @@ -44,15 +46,18 @@ def RepWarning(msg): parser.add_argument("--project", dest='project_name', help="Project name") parser.add_argument("--version", dest='version_name', help="Version name") -parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum projects to inspect else all") +parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum number of projects to inspect else all") parser.add_argument("--max-versions-per-project", dest='max_versions_per_project', type=int, help="Maximum versions per project to inspect else all") parser.add_argument("--max-components", dest='max_components', type=int, help="Maximum components to inspect in total else all") -parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none)") +parser.add_argument("--skip-projects", dest='skip_projects', type=int, help="Skip first 'n' projects to inspect") + +parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none 'n'=level)") +parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)") parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") -parser.add_argument("-t", "--timeout", default=15, type=int, help="Adjust the (HTTP) session timeout value (default: 15s)") -parser.add_argument("-r", "--retries", default=3, type=int, help="Adjust the number of retries on failure (default: 3)") +parser.add_argument("--timeout", default=60, type=int, help="Adjust the (HTTP) session timeout value (default: 60s)") +parser.add_argument("--retries", default=3, type=int, help="Adjust the number of retries on failure (default: 3)") args = parser.parse_args() @@ -63,8 +68,8 @@ def RepWarning(msg): # access the Black Duck platform bd = Client( base_url=args.base_url, - token=access_token, verify=args.verify, + token=access_token, timeout=args.timeout, retries=args.retries, ) @@ -73,6 +78,7 @@ def RepWarning(msg): all_my_comp_data = [] my_statistics = {} +str_unknown = "n/a" # version of components API to call comp_api_version = 6 @@ -118,7 +124,7 @@ def RepWarning(msg): my_statistics['_cntRefresh'] = 0 my_statistics['_cntNoOrigins'] = 0 my_statistics['_cntNoIDs'] = 0 - +my_statistics['_cntSkippedProjects'] = 0 # record any control values if args.project_name: @@ -149,18 +155,32 @@ def RepWarning(msg): # all projects are in scope projects = bd.get_resource('projects') + +cnt_projects = 0 + # loop through projects list for this_project in projects: + cnt_projects += 1 + + # check if we are skipping over this project + if args.skip_projects and cnt_projects <= args.skip_projects: + my_statistics['_cntSkippedProjects'] += 1 + RepDebug(1, 'Skipping project [%d] [%s]' % (cnt_projects, this_project['name'])) + continue + # check if we have hit any limit if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + RepDebug(1, 'Reached component limit [%d]' % args.max_components) break if args.max_projects and my_statistics['_cntProjects'] >= args.max_projects: + RepDebug(1, 'Reached project limit [%d]' % args.max_projects) break + # process this project my_statistics['_cntProjects'] += 1 - RepDebug(1, '## Project %d: %s' % (my_statistics['_cntProjects'], this_project['name'])) + RepDebug(1, '## Project: [%d] [%s]' % (cnt_projects, this_project['name'])) if args.version_name: # note the specific project version of interest @@ -181,11 +201,11 @@ def RepWarning(msg): # check if we have hit any limit if args.max_components and my_statistics['_cntComponents'] >= args.max_components: - # exit component loop - at the limit + RepDebug(1, 'Reached component limit [%d]' % args.max_components) break if args.max_versions_per_project and nVersionsPerProject >= args.max_versions_per_project: - # exit loop - at the version per project limit + RepDebug(1, 'Reached versions per project limit [%d]' % args.max_versions_per_project) break nVersionsPerProject += 1 @@ -193,66 +213,109 @@ def RepWarning(msg): # Announce # logging.debug(f"Found {this_project['name']}:{this_version['versionName']}") - RepDebug(3, ' Version: %s' % this_version['versionName']) + RepDebug(3, ' Version: [%s]' % this_version['versionName']) # iterate through all components for this project version for this_comp_data in bd.get_resource('components', this_version, **comp_kwargs): if args.max_components and my_statistics['_cntComponents'] >= args.max_components: - # exit component loop - at the limit + RepDebug(1, 'Reached component limit [%d]' % args.max_components) break my_statistics['_cntComponents'] += 1 - RepDebug(4, ' Component: %s (%s)' % - (this_comp_data['componentName'], this_comp_data['componentVersionName'])) + + if this_comp_data.get("componentName"): + comp_name = this_comp_data['componentName'] + else: + comp_name = str_unknown + + if this_comp_data.get("componentVersionName"): + comp_version_name = this_comp_data['componentVersionName'] + else: + comp_version_name = str_unknown + + comp_label = "{} ({})".format(comp_name, comp_version_name) + + RepDebug(4, ' Component: [%s]' % comp_label) if this_comp_data['inputExternalIds'].__len__() > 0: inputExternalIds = this_comp_data['inputExternalIds'][0] else: my_statistics['_cntNoIDs'] += 1 - inputExternalIds = "n/a" - RepDebug(2, ' ID: %s' % inputExternalIds) + inputExternalIds = str_unknown + RepDebug(2, ' ID: [%s]' % inputExternalIds) - # refresh the copyrights for this component + # refresh the copyrights for this component-origin if this_comp_data['origins'].__len__() > 0: - url = this_comp_data['origins'][0]['origin'] - else: - # no origins - RepWarning('No origin defined for [%s]' % this_comp_data['componentVersion']) -# url = this_comp_data['componentVersion'] - url = '' - - if len(url) > 0: - # refresh end point - url += "/copyrights-refresh" - - try: - response = bd.session.put(url, data=None, **refresh_kwargs) - RepDebug(5,'Refresh response %s' % response) - except urllib3.exceptions.ReadTimeoutError: - print('Failed to confirm copyrights refresh') - - my_statistics['_cntRefresh'] += 1 + + n_origin = 0 + + for this_origin in this_comp_data['origins']: + + n_origin += 1 + if this_origin.get('externalId'): + origin_id = this_origin['externalId'] + else: + origin_id = str_unknown + + url = this_origin['origin'] + + # refresh with end point + url += "/copyrights-refresh" + + status = -1 + + if args.dry_run != 0: + RepDebug(2, "DryRun: origin - no [%d] id [%s] url [%s]" % (n_origin, origin_id, url)) + else: + try: + response = bd.session.put(url, data=None, **refresh_kwargs) + RepDebug(5,'Refresh response: origin [%s] [%s]' % (this_origin, response)) + my_statistics['_cntRefresh'] += 1 + status= 0 + + except Exception: + print('Failed to confirm copyrights refresh') + status = 1 + + + # if recording the data - perhaps outputting to a CSV file + if args.dump_data: + my_data = {} + my_data['componentName'] = this_comp_data['componentName'] + my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['status'] = status + my_data['url'] = url + + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + + # add to our list + all_my_comp_data.append(my_data) + else: + # no origins defined + RepWarning('No origin(s) defined for [%s]' % comp_label) my_statistics['_cntNoOrigins'] += 1 + origin_id = '' + status = 3 url = 'n/a' + # if recording the data + if args.dump_data: + my_data = {} + my_data['componentName'] = comp_name + my_data['componentVersion'] = comp_version_name + my_data['status'] = status + my_data['url'] = url - # if recording the data - perhaps outputting to a CSV file - if args.dump_data: - my_data = {} - my_data['componentName'] = this_comp_data['componentName'] - my_data['componentVersion'] = this_comp_data['componentVersionName'] - my_data['url'] = url - - if hasattr(args, 'debug') and 5 <= args.debug: - pprint(my_data) - - # add to our list - all_my_comp_data.append(my_data) + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + # add to our list + all_my_comp_data.append(my_data) # end of processing loop @@ -275,6 +338,7 @@ def RepWarning(msg): field_names = [ 'Component', 'Component Version', + 'Status', 'Url' ] @@ -285,6 +349,7 @@ def RepWarning(msg): row_data = { 'Component': my_comp_data['componentName'], 'Component Version': my_comp_data['componentVersion'], + 'Status': my_comp_data['status'], 'Url': my_comp_data['url'] } writer.writerow(row_data) From 90f1db34a13621dcee7458444e77f58ccbda6697 Mon Sep 17 00:00:00 2001 From: Ian A <86781798+LuckySkyWalker@users.noreply.github.com> Date: Tue, 27 May 2025 15:06:31 +0100 Subject: [PATCH 140/146] Update refresh_project_copyrights.py Counting origins and clearer debug output --- examples/client/refresh_project_copyrights.py | 176 ++++++++++++------ 1 file changed, 120 insertions(+), 56 deletions(-) diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py index 4244b4b5..a6e9d3b7 100644 --- a/examples/client/refresh_project_copyrights.py +++ b/examples/client/refresh_project_copyrights.py @@ -4,6 +4,7 @@ # Ian Ashworth, May 2025 # import http.client +import signal from sys import api_version import sys import csv @@ -18,11 +19,18 @@ http.client._MAXHEADERS = 1000 +job_status = 0 + logging.basicConfig( level=logging.INFO, format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" ) +# initialise +all_my_comp_data = [] +my_statistics = {} + + def RepDebug(level, msg): if hasattr(args, 'debug') and level <= args.debug: print("dbg{" + str(level) + "} " + msg) @@ -33,6 +41,59 @@ def RepWarning(msg): print("WARNING: " + msg) return True +def CompleteTask(job_status): + now = datetime.datetime.now() + my_statistics['_jobStatus'] = job_status + + print('Finished: %s' % now.strftime("%Y-%m-%d %H:%M:%S")) + print('Summary:') + pprint(my_statistics) + + # if dumping data + if args.dump_data: + # if outputting to a CSV file + if args.csv_file: + '''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints + for a complete list of the fields available. The below code shows a subset of them just to + illustrate how to write out the data into a CSV format. + ''' + logging.info(f"Exporting {len(all_my_comp_data)} records to CSV file {args.csv_file}") + + with open(args.csv_file, 'w') as csv_f: + field_names = [ + 'Component', + 'Component Version', + 'Status', + 'Url' + ] + + writer = csv.DictWriter(csv_f, fieldnames=field_names) + writer.writeheader() + + for my_comp_data in all_my_comp_data: + row_data = { + 'Component': my_comp_data['componentName'], + 'Component Version': my_comp_data['componentVersion'], + 'Status': my_comp_data['status'], + 'Url': my_comp_data['url'] + } + writer.writerow(row_data) + else: + # print to screen + pprint(all_my_comp_data) + +def SignalHandler(sig, frame): + # Complete the work + print("Ctrl+C detected!") + + # tidy up and complete the job + CompleteTask(1) + sys.exit(job_status) + +# ------------------------------------------------------------------------------ +# register the signal handler +signal.signal(signal.SIGINT, SignalHandler) + # Parse command line arguments parser = argparse.ArgumentParser("Refresh copyrights for project/version components") @@ -46,10 +107,12 @@ def RepWarning(msg): parser.add_argument("--project", dest='project_name', help="Project name") parser.add_argument("--version", dest='version_name', help="Version name") -parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum projects to inspect else all") +parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum number of projects to inspect else all") parser.add_argument("--max-versions-per-project", dest='max_versions_per_project', type=int, help="Maximum versions per project to inspect else all") parser.add_argument("--max-components", dest='max_components', type=int, help="Maximum components to inspect in total else all") +parser.add_argument("--skip-projects", dest='skip_projects', type=int, help="Skip first 'n' projects to inspect") + parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none 'n'=level)") parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)") @@ -72,11 +135,10 @@ def RepWarning(msg): retries=args.retries, ) -# initialise -all_my_comp_data = [] -my_statistics = {} +str_unknown = "n/a" + # version of components API to call comp_api_version = 6 @@ -118,10 +180,13 @@ def RepWarning(msg): my_statistics['_cntProjects'] = 0 my_statistics['_cntVersions'] = 0 my_statistics['_cntComponents'] = 0 +my_statistics['_cntOrigins'] = 0 + my_statistics['_cntRefresh'] = 0 my_statistics['_cntNoOrigins'] = 0 my_statistics['_cntNoIDs'] = 0 - +my_statistics['_cntSkippedProjects'] = 0 +my_statistics['_jobStatus'] = 0 # record any control values if args.project_name: @@ -152,18 +217,33 @@ def RepWarning(msg): # all projects are in scope projects = bd.get_resource('projects') + +cnt_project = 0 +cnt_call = 0 + # loop through projects list for this_project in projects: + cnt_project += 1 + + # check if we are skipping over this project + if args.skip_projects and cnt_project <= args.skip_projects: + my_statistics['_cntSkippedProjects'] += 1 + RepDebug(1, 'Skipping project [%d] [%s]' % (cnt_project, this_project['name'])) + continue + # check if we have hit any limit if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + RepDebug(1, 'Reached component limit [%d]' % args.max_components) break if args.max_projects and my_statistics['_cntProjects'] >= args.max_projects: + RepDebug(1, 'Reached project limit [%d]' % args.max_projects) break + # process this project my_statistics['_cntProjects'] += 1 - RepDebug(1, '## Project %d: %s' % (my_statistics['_cntProjects'], this_project['name'])) + RepDebug(1, '## Project: [%d] [%s]' % (cnt_project, this_project['name'])) if args.version_name: # note the specific project version of interest @@ -184,11 +264,11 @@ def RepWarning(msg): # check if we have hit any limit if args.max_components and my_statistics['_cntComponents'] >= args.max_components: - # exit component loop - at the limit + RepDebug(1, 'Reached component limit [%d]' % args.max_components) break if args.max_versions_per_project and nVersionsPerProject >= args.max_versions_per_project: - # exit loop - at the version per project limit + RepDebug(1, 'Reached versions per project limit [%d]' % args.max_versions_per_project) break nVersionsPerProject += 1 @@ -196,30 +276,40 @@ def RepWarning(msg): # Announce # logging.debug(f"Found {this_project['name']}:{this_version['versionName']}") - RepDebug(3, ' Version: %s' % this_version['versionName']) + RepDebug(3, ' Version: [%s]' % this_version['versionName']) # iterate through all components for this project version for this_comp_data in bd.get_resource('components', this_version, **comp_kwargs): if args.max_components and my_statistics['_cntComponents'] >= args.max_components: - # exit component loop - at the limit break my_statistics['_cntComponents'] += 1 - comp_label = "{} ({})".format(this_comp_data['componentName'], this_comp_data['componentVersionName']) - RepDebug(4, ' Component: %s' % comp_label) + if this_comp_data.get("componentName"): + comp_name = this_comp_data['componentName'] + else: + comp_name = str_unknown + + if this_comp_data.get("componentVersionName"): + comp_version_name = this_comp_data['componentVersionName'] + else: + comp_version_name = str_unknown + + comp_label = "{} ({})".format(comp_name, comp_version_name) + + RepDebug(4, ' Component: [%s]' % comp_label) if this_comp_data['inputExternalIds'].__len__() > 0: inputExternalIds = this_comp_data['inputExternalIds'][0] else: my_statistics['_cntNoIDs'] += 1 - inputExternalIds = "n/a" - RepDebug(2, ' ID: %s' % inputExternalIds) + inputExternalIds = str_unknown + RepDebug(2, ' ID: [%s]' % inputExternalIds) - # refresh the copyrights for this component + # refresh the copyrights for this component-origin if this_comp_data['origins'].__len__() > 0: n_origin = 0 @@ -227,17 +317,27 @@ def RepWarning(msg): for this_origin in this_comp_data['origins']: n_origin += 1 - origin_id = this_origin['externalId'] + my_statistics['_cntOrigins'] += 1 + + if this_origin.get('externalId'): + origin_id = this_origin['externalId'] + else: + origin_id = str_unknown + url = this_origin['origin'] # refresh with end point url += "/copyrights-refresh" status = -1 + cnt_call += 1 + call_id = "{}.{}".format(cnt_project, cnt_call) if args.dry_run != 0: - RepDebug(1, "DryRun: no=%d origin=%s url=%s" % (n_origin, origin_id, url)) + RepDebug(2, ' DryRun: %s - origin - no [%d] id [%s] url [%s]' % (call_id, n_origin, origin_id, url)) else: + RepDebug(3, + ' Origin: %s - origin - no [%d] id [%s] url [%s]' % (call_id, n_origin, origin_id, url)) try: response = bd.session.put(url, data=None, **refresh_kwargs) RepDebug(5,'Refresh response: origin [%s] [%s]' % (this_origin, response)) @@ -274,8 +374,8 @@ def RepWarning(msg): # if recording the data if args.dump_data: my_data = {} - my_data['componentName'] = this_comp_data['componentName'] - my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['componentName'] = comp_name + my_data['componentVersion'] = comp_version_name my_data['status'] = status my_data['url'] = url @@ -287,42 +387,6 @@ def RepWarning(msg): # end of processing loop -now = datetime.datetime.now() -print('Finished: %s' % now.strftime("%Y-%m-%d %H:%M:%S")) -print('Summary:') -pprint(my_statistics) - -# if dumping data -if args.dump_data: - # if outputting to a CSV file - if args.csv_file: - '''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints - for a complete list of the fields available. The below code shows a subset of them just to - illustrate how to write out the data into a CSV format. - ''' - logging.info(f"Exporting {len(all_my_comp_data)} records to CSV file {args.csv_file}") - - with open(args.csv_file, 'w') as csv_f: - field_names = [ - 'Component', - 'Component Version', - 'Status', - 'Url' - ] - - writer = csv.DictWriter(csv_f, fieldnames=field_names) - writer.writeheader() - - for my_comp_data in all_my_comp_data: - row_data = { - 'Component': my_comp_data['componentName'], - 'Component Version': my_comp_data['componentVersion'], - 'Status': my_comp_data['status'], - 'Url': my_comp_data['url'] - } - writer.writerow(row_data) - else: - # print to screen - pprint(all_my_comp_data) +CompleteTask(0) #end From ee8513fb92f5ca0d049f25fc71e9eaa80eaa9bdf Mon Sep 17 00:00:00 2001 From: Mahesh Acharya Date: Mon, 4 Aug 2025 13:44:00 -0400 Subject: [PATCH 141/146] Update manage_project_structure.py added logic to skip base layer --- .../multi-image/manage_project_structure.py | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py index c648e63c..39b311e1 100644 --- a/examples/client/multi-image/manage_project_structure.py +++ b/examples/client/multi-image/manage_project_structure.py @@ -48,7 +48,7 @@ Sub-projects could be specified in a text file with -ssf --subproject-spec-file parameter Each line will have to contain full image specification. Specification will be parsed and image name prefixed by -str parameter will be used as a sub-project name -In this mode any image that is not residing on ciena.com repository will be skipped. +In this mode any image that is not residing on //your domain for example abc.com// repository will be skipped. If -str parameter is empty, Project Name will be used instead. Container image name scanned will be written into project version nickname field @@ -84,6 +84,7 @@ def __init__(self, args): else: self.init_project_data(args) self.serialize = args.serialize + self.skip_group=args.skip_group def connect(self): self.client = Client(base_url=self.base_url, token=self.access_token, verify=self.no_verify, timeout=60.0, retries=4) @@ -314,15 +315,17 @@ def process_text_spec_file(self,args): prefix = args.string_to_put_in_front_of_subproject_name if not prefix: prefix = args.project_name - with open(args.subproject_spec_file, "r") as f: - lines = f.read().splitlines() - for line in lines: - image_name = line.split('/')[-1].split(':')[0] # Don't look at me, you wrote it! - sub_project_name = "_".join((prefix, image_name)) - spec_line = ":".join((sub_project_name, line)) - # if "ciena.com" in spec_line: - project_list.append(spec_line) - return (project_list) + if args.subproject_spec_file is not None : + with open(args.subproject_spec_file, "r") as f: + lines = f.read().splitlines() + for line in lines: + #print(line) + image_name = line.split('/')[-1].split(':')[0] # Don't look at me, you wrote it! + sub_project_name = "_".join((prefix, image_name)) + spec_line = ":".join((sub_project_name, line)) + # if "//your domain ex. abc.com //" in spec_line: + project_list.append(spec_line) + return (project_list) def get_child_spec_list(self,args): if args.subproject_list: @@ -459,7 +462,8 @@ def scan_container_images(self): parent_version, detect_options, hub=hub, - binary=self.binary + binary=self.binary, + skip_group = self.skip_group ) child['scan_results'] = results except Exception as e: @@ -537,6 +541,7 @@ def parse_command_args(): parser.add_argument("-ifm", "--individual-file-matching", action='store_true', help="Turn Individual file matching on") parser.add_argument("--reprocess-run-file", help="Reprocess Failures from previous run report.") parser.add_argument("--serialize", action='store_true', help="Serialize scan submissions by adding --detect.wait.for.results=true to scan invocations") + parser.add_argument("--skip-group", required=False, help="exclude layers belog to specific groups, ex. 'base' ") args = parser.parse_args() if not args.reprocess_run_file and not (args.project_name and args.version_name): parser.error("[ -p/--project-name and -pv/--version-name ] or --reprocess-run-file are required") @@ -565,3 +570,5 @@ def main(): if __name__ == "__main__": sys.exit(main()) + + From b87283991f416d728eeb8cf74c938bde15b7cf5c Mon Sep 17 00:00:00 2001 From: Mahesh Acharya Date: Mon, 4 Aug 2025 13:47:03 -0400 Subject: [PATCH 142/146] Update scan_docker_image_lite.py logic added to skip base layer --- .../multi-image/scan_docker_image_lite.py | 73 +++++++++++-------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py index 8508685f..a771588b 100644 --- a/examples/client/multi-image/scan_docker_image_lite.py +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -229,7 +229,7 @@ class ContainerImageScanner(): def __init__( self, hub, container_image_name, workdir='/tmp/workdir', - grouping=None, base_image=None, dockerfile=None, detect_options=None): + grouping=None, base_image=None, dockerfile=None, detect_options=None, skip_group=None): self.hub = hub self.hub_detect = Detector(hub) self.docker = DockerWrapper(workdir) @@ -251,6 +251,11 @@ def __init__( if detect_options: self.extra_options = detect_options.split(" ") self.binary = False + if skip_group: + self.skip_group=skip_group.split(",") + else: + self.skip_group=[] + def prepare_container_image(self): self.docker.initdir() @@ -344,7 +349,7 @@ def process_container_image_by_base_image_info(self): layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(num) self.layers.append(layer) num = num + 1 - # print (json.dumps(self.layers, indent=4)) + #print (json.dumps(self.layers, indent=4)) def process_oci_container_image_by_user_defined_groups(self): self.manifest = self.docker.read_manifest() @@ -373,7 +378,7 @@ def process_oci_container_image_by_user_defined_groups(self): layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(layer['index']) if not layer.get('empty_layer', False): layer['path'] = layer_paths.pop(0) - # print (json.dumps(self.layers, indent=4)) + #print (json.dumps(self.layers, indent=4)) def get_group_name(self, groups, index): group_name = 'undefined' @@ -408,34 +413,39 @@ def process_oci_container_image(self): def submit_layer_scans(self): for layer in self.layers: - if not layer.get('empty_layer', False): - options = [] - options.append('--detect.project.name={}'.format(layer['project_name'])) - options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) - options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) - if self.binary: - options.append('--detect.tools=BINARY_SCAN') - options.append('--detect.binary.scan.file.path={}/{}'.format(self.docker.imagedir, layer['path'])) - else: - options.append('--detect.tools=SIGNATURE_SCAN') - if self.oci_layout: - options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'])) + + #print(f"layer group name={layer['group_name']} skip_group ={self.skip_group}") + + if layer['group_name'] not in self.skip_group: + + if not layer.get('empty_layer', False): + options = [] + options.append('--detect.project.name={}'.format(layer['project_name'])) + options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) + options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) + if self.binary: + options.append('--detect.tools=BINARY_SCAN') + options.append('--detect.binary.scan.file.path={}/{}'.format(self.docker.imagedir, layer['path'])) else: - options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) - if self.base_image or self.grouping or self.dockerfile: - options.extend(self.adorn_extra_options(layer)) - else: - options.extend(self.extra_options) - logging.debug(f"Submitting scan for {layer['name']}") - completed = self.hub_detect.detect_run(options) - scan_results = dict() - for key, value in vars(completed).items(): - if type(value) is bytes: - scan_results[key] = value.decode('utf-8') + options.append('--detect.tools=SIGNATURE_SCAN') + if self.oci_layout: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) + if self.base_image or self.grouping or self.dockerfile: + options.extend(self.adorn_extra_options(layer)) else: - scan_results[key] = value - layer['scan_results'] = scan_results - logging.debug(f"Detect run for {layer['name']} completed with returncode {completed.returncode}") + options.extend(self.extra_options) + logging.debug(f"Submitting scan for {layer['name']}") + completed = self.hub_detect.detect_run(options) + scan_results = dict() + for key, value in vars(completed).items(): + if type(value) is bytes: + scan_results[key] = value.decode('utf-8') + else: + scan_results[key] = value + layer['scan_results'] = scan_results + logging.debug(f"Detect run for {layer['name']} completed with returncode {completed.returncode}") def adorn_extra_options(self, layer): result = list() @@ -486,7 +496,7 @@ def get_base_layers(self): def scan_container_image( imagespec, grouping=None, base_image=None, dockerfile=None, - project_name=None, project_version=None, detect_options=None, hub=None, binary=False): + project_name=None, project_version=None, detect_options=None, hub=None, binary=False, skip_group=None ): if hub: hub = hub @@ -494,7 +504,7 @@ def scan_container_image( hub = HubInstance() scanner = ContainerImageScanner( hub, imagespec, grouping=grouping, base_image=base_image, - dockerfile=dockerfile, detect_options=detect_options) + dockerfile=dockerfile, detect_options=detect_options, skip_group=skip_group) if project_name: scanner.project_name = project_name if project_version: @@ -507,6 +517,7 @@ def scan_container_image( if binary: scanner.binary = True logging.info(f"Scanning image {imagespec}") + scanner.prepare_container_image() scanner.process_container_image() scanner.submit_layer_scans() From b08833833131230c6d23f4f99d76c5a016da65c5 Mon Sep 17 00:00:00 2001 From: Andrew Bolster Date: Wed, 3 Sep 2025 16:24:58 +0100 Subject: [PATCH 143/146] Add Model Context Protocol (MCP) integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add blackduck --mcp CLI command for MCP server - Implement MCP server with 6 BlackDuck tools - Add optional FastMCP dependency via [mcp] extra - Update README with MCP usage instructions - Maintain full backward compatibility 🤖 Generated with Claude Code Co-Authored-By: Claude --- README.md | 58 ++++++ blackduck/__main__.py | 72 ++++++++ blackduck/mcp_server.py | 399 ++++++++++++++++++++++++++++++++++++++++ setup.py | 12 +- 4 files changed, 538 insertions(+), 3 deletions(-) create mode 100644 blackduck/__main__.py create mode 100644 blackduck/mcp_server.py diff --git a/README.md b/README.md index bcab6abc..70a1c227 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,68 @@ Example code showing how to work with the new Client can be found in the *exampl **Examples which use the old HubInstance interface -which is not maintained- are not guaranteed to work. Use at your own risk.** +# MCP Integration + +The BlackDuck library now includes Model Context Protocol (MCP) support, enabling AI assistants to interact with BlackDuck Hub programmatically. + +## Installation with MCP Support + +```bash +pip3 install blackduck[mcp] +``` + +## Usage + +### 1. Set Environment Variables +```bash +export BLACKDUCK_URL="https://your.blackduck.url" +export BLACKDUCK_TOKEN="your-api-token" +``` + +### 2. Start MCP Server +```bash +blackduck --mcp +``` + +### 3. Configure MCP Client +For Claude Code, add to your MCP configuration: +```json +{ + "mcpServers": { + "blackduck": { + "command": "blackduck", + "args": ["--mcp"], + "env": { + "BLACKDUCK_URL": "https://your.blackduck.url", + "BLACKDUCK_TOKEN": "your-api-token" + } + } + } +} +``` + +### Available MCP Tools +- `list_projects` - List BlackDuck projects +- `get_project_details` - Get project information +- `list_project_versions` - List project versions +- `search_projects` - Search projects by name +- `get_project_vulnerabilities` - Get security vulnerabilities +- `list_project_components` - List project components + +See [MCP_INTEGRATION.md](MCP_INTEGRATION.md) for detailed documentation. + # Version History Including a version history on a go-forward basis. +## v1.1.4 (Upcoming) + +Added Model Context Protocol (MCP) integration: +- New `blackduck --mcp` CLI command to start MCP server +- Support for AI assistants via MCP tools +- Optional FastMCP dependency via `pip install blackduck[mcp]` +- Six core MCP tools for project and vulnerability management + ## v1.1.0 Retries will be attempted for all HTTP verbs, not just GET. diff --git a/blackduck/__main__.py b/blackduck/__main__.py new file mode 100644 index 00000000..fdd2b6a3 --- /dev/null +++ b/blackduck/__main__.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +""" +BlackDuck CLI entry point +""" + +import sys +import argparse +import os +import logging + +def main(): + """Main CLI entry point for BlackDuck""" + parser = argparse.ArgumentParser( + prog='blackduck', + description='BlackDuck Hub REST API CLI' + ) + + parser.add_argument( + '--mcp', + action='store_true', + help='Launch MCP (Model Context Protocol) server' + ) + + parser.add_argument( + '--version', + action='version', + version=f'blackduck {get_version()}' + ) + + args = parser.parse_args() + + if args.mcp: + launch_mcp_server() + else: + show_usage() + +def get_version(): + """Get package version""" + try: + from .__version__ import __version__ + return __version__ + except ImportError: + return 'unknown' + +def show_usage(): + """Show basic usage information""" + print("BlackDuck Hub REST API CLI") + print() + print("Usage:") + print(" blackduck --mcp Launch MCP server") + print(" blackduck --version Show version") + print() + print("For MCP server usage:") + print(" Set environment variables:") + print(" BLACKDUCK_URL") + print(" BLACKDUCK_TOKEN") + +def launch_mcp_server(): + """Launch the MCP server""" + try: + from .mcp_server import run_mcp_server + run_mcp_server() + except ImportError as e: + print(f"Error: MCP server dependencies not available: {e}", file=sys.stderr) + print("Install with: pip install fastmcp", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error launching MCP server: {e}", file=sys.stderr) + sys.exit(1) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/blackduck/mcp_server.py b/blackduck/mcp_server.py new file mode 100644 index 00000000..48ba55f1 --- /dev/null +++ b/blackduck/mcp_server.py @@ -0,0 +1,399 @@ +""" +BlackDuck MCP Server + +Provides Model Context Protocol interface for BlackDuck Hub REST API +""" + +import os +import sys +import logging +from typing import List, Dict, Any, Optional +import json + +# Configure logging for MCP server +logging.basicConfig(level=logging.WARNING) +logger = logging.getLogger(__name__) + +try: + from fastmcp import FastMCP +except ImportError: + raise ImportError( + "FastMCP not available. Install with: pip install fastmcp" + ) + +from .Client import Client +from .Utils import safe_get + + +class BlackDuckMCPServer: + """BlackDuck MCP Server implementation""" + + def __init__(self): + self.mcp = FastMCP("BlackDuck Hub") + self.client = None + self._setup_client() + self._register_tools() + + def _setup_client(self): + """Initialize BlackDuck client from environment variables""" + base_url = os.environ.get('BLACKDUCK_URL') + token = os.environ.get('BLACKDUCK_TOKEN') + + if not base_url or not token: + raise ValueError( + "Missing required environment variables:\n" + " BLACKDUCK_URL and BLACKDUCK_TOKEN" + ) + + self.client = Client( + base_url=base_url, + token=token, + verify=True, # Default to secure + timeout=30.0, + retries=3 + ) + + logger.info(f"BlackDuck client initialized for {base_url}") + + def _register_tools(self): + """Register MCP tools""" + + @self.mcp.tool + def list_projects(limit: Optional[int] = 50) -> List[Dict[str, Any]]: + """List BlackDuck projects + + Args: + limit: Maximum number of projects to return (default: 50) + + Returns: + List of project dictionaries with name, description, and metadata + """ + try: + projects = self.client.get_resource('projects') + result = [] + + for i, project in enumerate(projects): + if limit and i >= limit: + break + + result.append({ + 'name': project.get('name'), + 'description': project.get('description', ''), + 'projectOwner': safe_get(project, 'projectOwner'), + 'createdAt': project.get('createdAt'), + 'updatedAt': project.get('updatedAt'), + '_meta': { + 'href': project.get('_meta', {}).get('href') + } + }) + + return result + + except Exception as e: + logger.error(f"Error listing projects: {e}") + raise + + @self.mcp.tool + def get_project_details(project_name: str) -> Optional[Dict[str, Any]]: + """Get detailed information about a specific project + + Args: + project_name: Name of the project to retrieve + + Returns: + Project details dictionary or None if not found + """ + try: + params = {'q': [f"name:{project_name}"]} + projects = list(self.client.get_resource('projects', params=params)) + + # Find exact match (case-insensitive) + project = None + for p in projects: + if p['name'].lower() == project_name.lower(): + project = p + break + + if not project: + return None + + return { + 'name': project.get('name'), + 'description': project.get('description', ''), + 'projectOwner': safe_get(project, 'projectOwner'), + 'createdAt': project.get('createdAt'), + 'updatedAt': project.get('updatedAt'), + 'projectLevelAdjustments': project.get('projectLevelAdjustments', False), + 'cloneCategories': project.get('cloneCategories', []), + '_meta': project.get('_meta', {}) + } + + except Exception as e: + logger.error(f"Error getting project details: {e}") + raise + + @self.mcp.tool + def list_project_versions(project_name: str, limit: Optional[int] = 20) -> List[Dict[str, Any]]: + """List versions for a specific project + + Args: + project_name: Name of the project + limit: Maximum number of versions to return (default: 20) + + Returns: + List of version dictionaries + """ + try: + # First find the project + params = {'q': [f"name:{project_name}"]} + projects = list(self.client.get_resource('projects', params=params)) + + project = None + for p in projects: + if p['name'].lower() == project_name.lower(): + project = p + break + + if not project: + return [] + + # Get versions for the project + versions = self.client.get_resource('versions', project) + result = [] + + for i, version in enumerate(versions): + if limit and i >= limit: + break + + result.append({ + 'versionName': version.get('versionName'), + 'nickname': version.get('nickname'), + 'phase': version.get('phase'), + 'distribution': version.get('distribution'), + 'createdAt': version.get('createdAt'), + 'settingUpdatedAt': version.get('settingUpdatedAt'), + '_meta': { + 'href': version.get('_meta', {}).get('href') + } + }) + + return result + + except Exception as e: + logger.error(f"Error listing project versions: {e}") + raise + + @self.mcp.tool + def search_projects(query: str, limit: Optional[int] = 25) -> List[Dict[str, Any]]: + """Search for projects by name or description + + Args: + query: Search query string + limit: Maximum number of results to return (default: 25) + + Returns: + List of matching project dictionaries + """ + try: + params = {'q': [f"name:{query}"]} + projects = self.client.get_resource('projects', params=params) + result = [] + + for i, project in enumerate(projects): + if limit and i >= limit: + break + + # Simple relevance scoring + name = project.get('name', '').lower() + description = project.get('description', '').lower() + query_lower = query.lower() + + relevance = 0 + if query_lower in name: + relevance += 2 + if query_lower in description: + relevance += 1 + + result.append({ + 'name': project.get('name'), + 'description': project.get('description', ''), + 'relevance': relevance, + 'createdAt': project.get('createdAt'), + '_meta': { + 'href': project.get('_meta', {}).get('href') + } + }) + + # Sort by relevance + result.sort(key=lambda x: x['relevance'], reverse=True) + return result + + except Exception as e: + logger.error(f"Error searching projects: {e}") + raise + + @self.mcp.tool + def get_project_vulnerabilities(project_name: str, version_name: Optional[str] = None, limit: Optional[int] = 50) -> List[Dict[str, Any]]: + """Get vulnerabilities for a project version + + Args: + project_name: Name of the project + version_name: Name of the version (if None, uses latest) + limit: Maximum number of vulnerabilities to return (default: 50) + + Returns: + List of vulnerability dictionaries + """ + try: + # Find project + params = {'q': [f"name:{project_name}"]} + projects = list(self.client.get_resource('projects', params=params)) + + project = None + for p in projects: + if p['name'].lower() == project_name.lower(): + project = p + break + + if not project: + return [] + + # Find version + versions = list(self.client.get_resource('versions', project)) + if not versions: + return [] + + version = None + if version_name: + for v in versions: + if v['versionName'].lower() == version_name.lower(): + version = v + break + else: + # Use first (most recent) version + version = versions[0] + + if not version: + return [] + + # Get vulnerabilities + try: + vulnerabilities = self.client.get_resource('vulnerable-components', version) + result = [] + + for i, vuln in enumerate(vulnerabilities): + if limit and i >= limit: + break + + result.append({ + 'componentName': vuln.get('componentName'), + 'componentVersionName': vuln.get('componentVersionName'), + 'vulnerabilityName': vuln.get('vulnerabilityName'), + 'severity': vuln.get('severity'), + 'baseScore': vuln.get('baseScore'), + 'overallScore': vuln.get('overallScore'), + 'remediationStatus': vuln.get('remediationStatus'), + 'description': vuln.get('description', ''), + 'publishedDate': vuln.get('publishedDate'), + 'updatedDate': vuln.get('updatedDate') + }) + + return result + + except Exception: + # Fallback: try to get components instead + components = self.client.get_resource('components', version) + return [{'info': 'Use list_project_components for component information'}] + + except Exception as e: + logger.error(f"Error getting vulnerabilities: {e}") + raise + + @self.mcp.tool + def list_project_components(project_name: str, version_name: Optional[str] = None, limit: Optional[int] = 50) -> List[Dict[str, Any]]: + """List components in a project version + + Args: + project_name: Name of the project + version_name: Name of the version (if None, uses latest) + limit: Maximum number of components to return (default: 50) + + Returns: + List of component dictionaries + """ + try: + # Find project + params = {'q': [f"name:{project_name}"]} + projects = list(self.client.get_resource('projects', params=params)) + + project = None + for p in projects: + if p['name'].lower() == project_name.lower(): + project = p + break + + if not project: + return [] + + # Find version + versions = list(self.client.get_resource('versions', project)) + if not versions: + return [] + + version = None + if version_name: + for v in versions: + if v['versionName'].lower() == version_name.lower(): + version = v + break + else: + version = versions[0] + + if not version: + return [] + + # Get components + components = self.client.get_resource('components', version) + result = [] + + for i, component in enumerate(components): + if limit and i >= limit: + break + + licenses = component.get('licenses', []) + license_display = licenses[0].get('licenseDisplay', 'Unknown') if licenses else 'Unknown' + + result.append({ + 'componentName': component.get('componentName'), + 'componentVersionName': component.get('componentVersionName'), + 'matchTypes': component.get('matchTypes', []), + 'usages': component.get('usages', []), + 'licenseDisplay': license_display, + 'policyStatus': component.get('policyStatus'), + 'securityRiskProfile': component.get('securityRiskProfile'), + 'activityData': component.get('activityData') + }) + + return result + + except Exception as e: + logger.error(f"Error listing components: {e}") + raise + + def run(self): + """Run the MCP server""" + self.mcp.run() + + +def run_mcp_server(): + """Entry point for running the MCP server""" + try: + server = BlackDuckMCPServer() + server.run() + except Exception as e: + logger.error(f"Failed to start MCP server: {e}") + sys.exit(1) + + +if __name__ == '__main__': + run_mcp_server() \ No newline at end of file diff --git a/setup.py b/setup.py index 7eadcc73..51ad8d13 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,11 @@ 'requests', 'python-dateutil' ] +# Optional dependencies for MCP server +EXTRAS = { + 'mcp': ['fastmcp'] +} + # The rest you shouldn't have to touch too much :) # ------------------------------------------------ # Except, perhaps the License and Trove Classifiers! @@ -100,9 +105,9 @@ def run(self): # If your package is a single module, use this instead of 'packages': # py_modules=['mypackage'], - # entry_points={ - # 'console_scripts': ['mycli=mymodule:cli'], - # }, + entry_points={ + 'console_scripts': ['blackduck=blackduck.__main__:main'], + }, # Seems like dependency_links is potentiall going away. That said, using pip 10 # I was able to install using 'pip install dist/hubpy-0.0.1.tar.gz --process-dependency-links' @@ -111,6 +116,7 @@ def run(self): # 'git+git://github.com/blackducksoftware/tortilla#egg=tortilla-0.5.1b' # ], install_requires=REQUIRED, + extras_require=EXTRAS, setup_requires=['pytest-runner'], tests_require=['pytest', 'requests-mock', 'pytest-datadir'], include_package_data=True, From 998eee78732135ac8850e10e202ac29e60af58eb Mon Sep 17 00:00:00 2001 From: Dinesh Ravi Date: Tue, 27 Jan 2026 06:54:57 +0100 Subject: [PATCH 144/146] Add SPDX_30 and CYCLONEDX_16 options for SBOM type --- examples/client/generate_sbom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/client/generate_sbom.py b/examples/client/generate_sbom.py index f2da3249..d256d152 100644 --- a/examples/client/generate_sbom.py +++ b/examples/client/generate_sbom.py @@ -45,7 +45,7 @@ class FailedReportDownload(Exception): parser.add_argument("token_file", help="containing access token") parser.add_argument("project_name") parser.add_argument("version_name") -parser.add_argument("-t", "--type", type=str, nargs='?', default="SPDX_23", choices=["SPDX_22", "SPDX_23", "CYCLONEDX_13", "CYCLONEDX_14"], help="Choose the type of SBOM report") +parser.add_argument("-t", "--type", type=str, nargs='?', default="SPDX_23", choices=["SPDX_22", "SPDX_23","SPDX_30", "CYCLONEDX_13", "CYCLONEDX_14","CYCLONEDX_16", "CYCLONEDX_15"], help="Choose the type of SBOM report") parser.add_argument('-r', '--retries', default=4, type=int, help="How many times to retry downloading the report, i.e. wait for the report to be generated") parser.add_argument('-s', '--sleep_seconds', default=60, type=int, help="The amount of time to sleep in-between (re-)tries to download the report") parser.add_argument('--include-subprojects', dest='include_subprojects', action='store_false', help="whether subprojects should be included") From cf571f210bafffa609bfddb6b37be6f2513871b2 Mon Sep 17 00:00:00 2001 From: Andrew Bolster Date: Thu, 28 May 2026 21:42:46 +0100 Subject: [PATCH 145/146] chore: pin dependencies with hashes and add CI test workflow (#295) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: add Polaris SAST/SCA scanning Add Polaris security scanning configuration and GitHub Actions workflow for SAST and SCA analysis of the hub-rest-api-python project. Co-Authored-By: Claude Sonnet 4.6 * chore: pin dependencies with hashes and add CI test workflow Supply chain hardening: - Pin all deps in requirements.txt to exact == versions (requests bumped from 2.31.0 to 2.32.4 to address known vulnerability CVE-2024-35195) - Add requirements.lock.txt generated by pip-compile --generate-hashes, providing full transitive dependency closure with sha256 hashes for tamper-evident installs - Add CI workflow (.github/workflows/ci.yml) that installs via pip install --require-hashes -r requirements.lock.txt and runs pytest across Python 3.9–3.12, with a lockfile presence check - Update codeql-analysis.yml to use SHA-pinned action refs (v1 → v4/v3) to prevent dependency confusion via mutable action tags To regenerate the lockfile after changing requirements.txt: pip-compile --generate-hashes --allow-unsafe requirements.txt -o requirements.lock.txt Co-Authored-By: Claude Sonnet 4.6 * Remove Polaris scanning config — not appropriate for this upstream repo polaris-scan.yml, polaris.yml, and the .gitignore additions are specific to the Data Science team's internal tooling and should not be included in the hub-rest-api-python upstream contribution. Co-Authored-By: Claude Sonnet 4.6 * Fix trailing blank line in .gitignore Co-Authored-By: Claude Sonnet 4.6 --------- Co-authored-by: Claude Sonnet 4.6 --- .github/workflows/ci.yml | 31 ++ .github/workflows/codeql-analysis.yml | 8 +- requirements.lock.txt | 474 ++++++++++++++++++++++++++ requirements.txt | 4 +- 4 files changed, 511 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 requirements.lock.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..fb06ca0b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Verify lockfile present + run: test -f requirements.lock.txt || (echo "requirements.lock.txt missing — run pip-compile --generate-hashes requirements.txt -o requirements.lock.txt and commit it" && exit 1) + + - name: Install dependencies (hash-verified) + run: pip install --require-hashes -r requirements.lock.txt + + - name: Run tests + run: pytest test/ diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 953e4167..f74b13c6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -24,15 +24,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@051e2f90686233507fe9283ff167d2e709304b30 # v3.36.0 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@051e2f90686233507fe9283ff167d2e709304b30 # v3.36.0 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@051e2f90686233507fe9283ff167d2e709304b30 # v3.36.0 diff --git a/requirements.lock.txt b/requirements.lock.txt new file mode 100644 index 00000000..c18b3288 --- /dev/null +++ b/requirements.lock.txt @@ -0,0 +1,474 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --no-emit-trusted-host --output-file=requirements.lock.txt --strip-extras requirements.txt +# +arrow==1.4.0 \ + --hash=sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205 \ + --hash=sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7 + # via -r requirements.txt +certifi==2026.5.20 \ + --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ + --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d + # via requests +charset-normalizer==3.4.7 \ + --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ + --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ + --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ + --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ + --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ + --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ + --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ + --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ + --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ + --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ + --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ + --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ + --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ + --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ + --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ + --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ + --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ + --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ + --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ + --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ + --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ + --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ + --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ + --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ + --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ + --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ + --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ + --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ + --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ + --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ + --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ + --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ + --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ + --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ + --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ + --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ + --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ + --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ + --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ + --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ + --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ + --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ + --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ + --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ + --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ + --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ + --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ + --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ + --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ + --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ + --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ + --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ + --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ + --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ + --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ + --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ + --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ + --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ + --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ + --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ + --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ + --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ + --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ + --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ + --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ + --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ + --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ + --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ + --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ + --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ + --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ + --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ + --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ + --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ + --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ + --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ + --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ + --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ + --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ + --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ + --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ + --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ + --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ + --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ + --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ + --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ + --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ + --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ + --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ + --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ + --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ + --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ + --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ + --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ + --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ + --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ + --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ + --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ + --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ + --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ + --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ + --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ + --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ + --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ + --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ + --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ + --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ + --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ + --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ + --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ + --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ + --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ + --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ + --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ + --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ + --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ + --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ + --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ + --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ + --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ + --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ + --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ + --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ + --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ + --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ + --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ + --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ + --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 + # via requests +docutils==0.23 \ + --hash=sha256:25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea \ + --hash=sha256:746f5060322511280a1e50eb76846ed6bf2342984b2ac04dc42caa1a8d78799e + # via readme-renderer +id==1.6.1 \ + --hash=sha256:279ec98b49c315880403d0e87b218e7d4e08c9c487992395a6b4498677042d47 \ + --hash=sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069 \ + --hash=sha256:f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca + # via twine +idna==3.16 \ + --hash=sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5 \ + --hash=sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d + # via requests +iniconfig==2.3.0 \ + --hash=sha256:bd930604c1d2d3ef15b8cabe666358162e51874f80ad784aa61329c0d0bd8362 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +jaraco-classes==3.4.0 \ + --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ + --hash=sha256:828b4dde7f79f21d3ea9c933f393f6bdbd55e6303ed1cb1f2377874681939b6d \ + --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 + # via keyring +jaraco-context==6.1.2 \ + --hash=sha256:7e9523faf738ebac6618a69251efeed20d4bbf25adf2add8eb0822278ca84c07 \ + --hash=sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535 \ + --hash=sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3 + # via keyring +jaraco-functools==4.5.0 \ + --hash=sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03 \ + --hash=sha256:79ce39246eddbde4b3a03b77ea5f0f7878dc669b166a66cf3fa8e266aa3fa2f4 + # via keyring +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via pytest-html +keyring==25.7.0 \ + --hash=sha256:20c6e6b97380f809fb2f63f49b019cd04a221139319244c20758310104903974 \ + --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f \ + --hash=sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b + # via twine +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:746657e7635cdfa843d6ec4dc7ab87bd57ce5f13a9602ff58af2b57295842c92 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:93ad242e81180c5d119b52461ee4b364aeaab9454bdff49d4b411715d6f74f99 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +more-itertools==11.1.0 \ + --hash=sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d \ + --hash=sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 + # via + # jaraco-classes + # jaraco-functools +nh3==0.3.5 \ + --hash=sha256:005974514896b7e0cb9437876bebd7c6eb83b514a09c26cd74cae3d7c02e8ca4 \ + --hash=sha256:0a09f51806fd51b4fedbf9ea2b61fef388f19aef0d62fe51199d41648be14588 \ + --hash=sha256:0ef66a32c8fa957406e61b745d02f4b6dc9ad596f724a4c97aa37ea7bfb156c6 \ + --hash=sha256:1bb0ec0f3e9eea439f5f40dc241a7790cb8edd308bd48d31d85cf63dd68ba3d2 \ + --hash=sha256:207c01801d3e9bb8ec08f08689346bdd30ce15b8bf60013a925d08b5388962a4 \ + --hash=sha256:23a312224875f72cd16bde417f49071451877e29ef646a60e50fcb69407cc18a \ + --hash=sha256:2c069570b06aa848457713ad7af4a9905691291548c4466a9ad78ee95808382b \ + --hash=sha256:33c7793c1e5fbb8c332dabfda33fb7ee9c9257764575ffd2b727d253136c1fc0 \ + --hash=sha256:38748140bf76383ab7ce2dce0ad4cb663855d8fbc9098f7f3483673d09616a17 \ + --hash=sha256:387abd011e81959d5a35151a11350a0795c6edeb53ebfa02d2e882dc01299263 \ + --hash=sha256:3bb854485c9b33e5bb143ff3e49e577073bc6bc320f0ff8fc316dd89c0d3c101 \ + --hash=sha256:45855e14ff056064fec77133bfcf7cd691838168e5e17bbef075394954dc9dc8 \ + --hash=sha256:45e6a65dc88a300a2e3502cb9c8e6d1d6b831d6fba7470643333609c6aab1f30 \ + --hash=sha256:488928988caad25ba14b1eb5bc74e25e21f3b5e40341d956f3ce4a8bc19460dc \ + --hash=sha256:48f45e3e914be93a596431aa143dedf1582557bf41a58153c296048d6e3798c9 \ + --hash=sha256:505b57235390029a9608e98e493520540fc5349c70c94f19e39e1553b452bf44 \ + --hash=sha256:50d401ab2d8e86d59e2126e3ab2a2f45840c405842b626d9a51624b3a33b6878 \ + --hash=sha256:52d877980d7ca01dc3baf3936bf844828bc6f332962227a684ed79c18cce14c3 \ + --hash=sha256:53758dd9f91757e50fbbd0e34e269933af2b9f758351e35283acc06cf33d9afd \ + --hash=sha256:559e4c73b689e9a7aa97ac9760b1bc488038d7c1a575aa4ab5a0e19ee9630c0f \ + --hash=sha256:694b0b0b71871d04e0c24897d841dae9c007e97a653e48d68818e175007e1571 \ + --hash=sha256:6dfe85a907cf2b650312c75d7740f18ffb956dd8bc64eb7d7d625d4f265e9dd4 \ + --hash=sha256:6ea58cc44d274c643b83547ca9654a0b1a817609b160601356f76a2b744c49ad \ + --hash=sha256:72c5bdedec27fa33de6a5326346ea8aa3fe54f6ac294d54c4b204fb66a9f1e79 \ + --hash=sha256:7989e36f0170d44da9268ffda95935f91621c03f341f6e744b29deadee1b9c66 \ + --hash=sha256:84bdeb082544fbcb77a12c034dd77d7da0556fdc0727b787eb6214b958c15e29 \ + --hash=sha256:84e3047ddb57fe6692f321c9c556d4ce22405a84994ecedb4c3b682a09e5426d \ + --hash=sha256:869828201c6a4264133d5b24503b7301681566594f4ad37601c43c1cb4ac0b4c \ + --hash=sha256:8f85285700a18e9f3fc5bff41fe573fa84f81542ef13b48a89f9fecca0474d3b \ + --hash=sha256:937847a57a9cf5d86e1379a0dcccf16e72925a8df11c0ec67ec3ec92de7c9749 \ + --hash=sha256:94da8643b5aad56be378f3dc7be19f59be8e34877ecdc6dccc19ca2859efbac0 \ + --hash=sha256:a5e4a2dd9210e005fb4dd8a6d18c589c99002f540b59b03a22a321f1843c0ae6 \ + --hash=sha256:a9d63bdd9f0dabb0df4ee180b27837eaba0b04a49c721d8c8be168bff5bd4892 \ + --hash=sha256:acfd354e61accbe4c74f8017c6e397a776916dfe47c48643cf7fd84ade826f93 \ + --hash=sha256:bb9ae0bd69cbf2410691982341aced596492a37f12a733787999b8b74ad644e2 \ + --hash=sha256:bc579e7a5bfad44d7b9f2bb6b12fcdfb19124dc0b38b2d6aeafd5536da4e5c71 \ + --hash=sha256:c187635a0a39f5680d66a8226451d4fbbf3d905cca4646ef17e62a0f9d70cdc6 \ + --hash=sha256:c357f1d042c67f135a5e6babb2b0e3b9d9224ff4a3543240f597767b01384ffd \ + --hash=sha256:c3aae321f67ae66cff2a627115f106a377d4475d10b0e13d97959a13486b9a88 \ + --hash=sha256:c88605d8d468f7fc1b31e06129bc91d6c96f6c621776c9b504a0da9beac9df5f \ + --hash=sha256:c9ce78ce384fd4865cc0c0a662df34fc452ec75f4c94706c82d3996e9a395447 \ + --hash=sha256:c9d1d8dfb01a8dca2fe3aa758432baa006c0be32305c4f2f7198ce2a1537d244 \ + --hash=sha256:d74f4345cfef7298607af6d46db26dcd89885f49335159d38c36291b0ca16a8a \ + --hash=sha256:de8e8621853b6470fe928c684ee0d3f39ea8086cebafe4c416486488dea7b68d \ + --hash=sha256:e49c9b564e6bcb03ecd2f057213df9a0de15a95812ac9db9600b590db23d3ae9 \ + --hash=sha256:ea232933394d1d58bf7c4bb348dc4660eae6604e1ae81cd2ba6d9ed80d390f3b \ + --hash=sha256:eeedc90ed8c42c327e8e10e621ccfa314fc6cce35d5929f4297ff1cdb89667c4 \ + --hash=sha256:f5d471d436a6147c7a8075a2294f37f628318a5f7945489ddfa8605fb7641fc1 \ + --hash=sha256:fe3a787dc76b50de6bee54ef242f26c41dfe47654428e3e94f0fae5bb6dd2cc1 + # via readme-renderer +packaging==26.2 \ + --hash=sha256:41a77245620462d3c4fe6cdc2c61f01dee0e93180ce554120d92b0722575b3cf \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 + # via + # pytest + # twine + # wheel +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:dd599d9a0d5fbd9584da20ea661495e778e008401ce99f76963c1da2fc2c3f1d \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 \ + --hash=sha256:d3819ae5d79ed1f4347cd6baa51210b13ebcd6965b5e981a6cac3295acb63894 + # via + # pytest + # readme-renderer + # rich +pytest==9.0.3 \ + --hash=sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9 \ + --hash=sha256:b7e6b08772077d35e611ef43c30d186001e5b84c36210b4ff651acfb90f07408 \ + --hash=sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c + # via + # -r requirements.txt + # pytest-datadir + # pytest-html + # pytest-metadata +pytest-datadir==1.8.0 \ + --hash=sha256:5c677bc097d907ac71ca418109adc3abe34cf0bddfe6cf78aecfbabd96a15cf0 \ + --hash=sha256:7a15faed76cebe87cc91941dd1920a9a38eba56a09c11e9ddf1434d28a0f78eb + # via -r requirements.txt +pytest-html==4.2.0 \ + --hash=sha256:4dd34e9a332e96ba7fb140b68632258bec573d2c43166aa671a94986f72bbf7b \ + --hash=sha256:b6a88cba507500d8709959201e2e757d3941e859fd17cfd4ed87b16fc0c67912 \ + --hash=sha256:ff5caf3e17a974008e5816edda61168e6c3da442b078a44f8744865862a85636 + # via -r requirements.txt +pytest-metadata==3.1.1 \ + --hash=sha256:9011d5355e01c1e212b49faeba8831bffab9940d11fc833973fa556bf6fd7ccd \ + --hash=sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b \ + --hash=sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8 + # via pytest-html +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via + # -r requirements.txt + # arrow +pytz==2026.2 \ + --hash=sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126 \ + --hash=sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a \ + --hash=sha256:fa3d37e583c69b4a745dd92ad835f996407448e86c31c31825159ed8c36eb5fa + # via timestring +readme-renderer==44.0 \ + --hash=sha256:2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151 \ + --hash=sha256:8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1 \ + --hash=sha256:950609fbe5e9d9b6ae18770a2b896cbad7194ee763377bdf10b22ee910c9df40 + # via twine +requests==2.32.4 \ + --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ + --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 \ + --hash=sha256:acc14812958ea21ca50b8bab45099264b3d7feba361acf40ddafd78832b05b98 + # via + # -r requirements.txt + # requests-mock + # requests-toolbelt + # twine +requests-mock==1.12.1 \ + --hash=sha256:3e8db63dff19719391b8c7c678d9e973e582361eeab0b3f80fa9228add556ca6 \ + --hash=sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563 \ + --hash=sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401 + # via -r requirements.txt +requests-toolbelt==1.0.0 \ + --hash=sha256:392594a42d9a9f291b9f04af37559fceadac8d3f8ce1ff6045f4922d36426c32 \ + --hash=sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6 \ + --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 + # via twine +rfc3986==2.0.0 \ + --hash=sha256:482a9b7950ab1c8458d321d4d9fc62e9e7e2bf5982378d552d4d36a950e9cc5b \ + --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ + --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c + # via twine +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:5dd8ecaf5d0da3119949338c3a4a70894a82c2509f6acd0f7d6916dbc36bd24f \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via twine +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +terminaltables==3.1.10 \ + --hash=sha256:ba6eca5cb5ba02bba4c9f4f985af80c54ec3dccf94cfcd190154386255e47543 \ + --hash=sha256:e4fdc4179c9e4aab5f674d80f09d76fa436b96fdc698a8505e0a36bf0804a874 + # via -r requirements.txt +timestring==1.6.4 \ + --hash=sha256:5c80d8cc555fd31d4d743acb4f028c9ddea294b247cdb407afb7ca050d5c4216 \ + --hash=sha256:b7d1a2060a5f2e34d0c93b1042690074afcea66638fa2cf4dcad2d1bec4deacc + # via -r requirements.txt +twine==6.2.0 \ + --hash=sha256:3da303f52d63484e824bfb792df1a2c92e849c6d5e1feffc7fb81c71def4e924 \ + --hash=sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8 \ + --hash=sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf + # via -r requirements.txt +tzdata==2026.2 \ + --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \ + --hash=sha256:93e00b2b8e83bd91fa05a073dc9d71a565e9c3c9eef0889ebedd978dab0c5f08 \ + --hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7 + # via arrow +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + # via + # id + # requests + # twine +wheel==0.47.0 \ + --hash=sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced \ + --hash=sha256:66c570b5f1e7276af684b2d1f11070dd332e0d2b78d8f649757d377cd0b211b0 \ + --hash=sha256:cc72bd1009ba0cf63922e28f94d9d83b920aa2bb28f798a31d0691b02fa3c9b3 + # via -r requirements.txt + +# The following packages are considered to be unsafe in a requirements file: +setuptools==82.0.1 \ + --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ + --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb \ + --hash=sha256:f2f48a947374136a39e9f349cf28f1900765657b892322e7ead81d0f466db134 + # via -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 6ee96357..69c138a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # For the library itself -requests==2.31.0 -python-dateutil>=2.8.0 +requests==2.32.4 +python-dateutil==2.9.0.post0 # for examples printing tables to the terminal terminaltables From 5ac8d1dd103fb0585a204621abf3732bb7a12a17 Mon Sep 17 00:00:00 2001 From: Andrew Bolster Date: Fri, 29 May 2026 14:23:49 +0100 Subject: [PATCH 146/146] fix(deps): add hashed lock file for html-notices example The examples/generate_html_notices_report_from_json/requirements.txt had jinja2 unpinned. Add a uv-generated lock file pinning jinja2==3.1.6 and its transitive dependency markupsafe==3.0.3, both with sha256 hashes. The root requirements.lock.txt (covering 11 previously unpinned packages: python-dateutil, terminaltables, timestring, pytest, pytest-html, requests-mock, pytest-datadir, setuptools, wheel, twine, arrow) was already committed in a prior change; this commit completes the supply chain fix for the example subdirectory. Co-Authored-By: Claude Sonnet 4.6 --- .../requirements.lock.txt | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 examples/generate_html_notices_report_from_json/requirements.lock.txt diff --git a/examples/generate_html_notices_report_from_json/requirements.lock.txt b/examples/generate_html_notices_report_from_json/requirements.lock.txt new file mode 100644 index 00000000..fba034f0 --- /dev/null +++ b/examples/generate_html_notices_report_from_json/requirements.lock.txt @@ -0,0 +1,97 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile examples/generate_html_notices_report_from_json/requirements.txt --generate-hashes -o examples/generate_html_notices_report_from_json/requirements.lock.txt +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via -r examples/generate_html_notices_report_from_json/requirements.txt +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2