diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..fb06ca0b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Verify lockfile present + run: test -f requirements.lock.txt || (echo "requirements.lock.txt missing — run pip-compile --generate-hashes requirements.txt -o requirements.lock.txt and commit it" && exit 1) + + - name: Install dependencies (hash-verified) + run: pip install --require-hashes -r requirements.lock.txt + + - name: Run tests + run: pytest test/ diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 953e4167..f74b13c6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -24,15 +24,15 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@051e2f90686233507fe9283ff167d2e709304b30 # v3.36.0 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@051e2f90686233507fe9283ff167d2e709304b30 # v3.36.0 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@051e2f90686233507fe9283ff167d2e709304b30 # v3.36.0 diff --git a/README.md b/README.md index bcab6abc..70a1c227 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,68 @@ Example code showing how to work with the new Client can be found in the *exampl **Examples which use the old HubInstance interface -which is not maintained- are not guaranteed to work. Use at your own risk.** +# MCP Integration + +The BlackDuck library now includes Model Context Protocol (MCP) support, enabling AI assistants to interact with BlackDuck Hub programmatically. + +## Installation with MCP Support + +```bash +pip3 install blackduck[mcp] +``` + +## Usage + +### 1. Set Environment Variables +```bash +export BLACKDUCK_URL="https://your.blackduck.url" +export BLACKDUCK_TOKEN="your-api-token" +``` + +### 2. Start MCP Server +```bash +blackduck --mcp +``` + +### 3. Configure MCP Client +For Claude Code, add to your MCP configuration: +```json +{ + "mcpServers": { + "blackduck": { + "command": "blackduck", + "args": ["--mcp"], + "env": { + "BLACKDUCK_URL": "https://your.blackduck.url", + "BLACKDUCK_TOKEN": "your-api-token" + } + } + } +} +``` + +### Available MCP Tools +- `list_projects` - List BlackDuck projects +- `get_project_details` - Get project information +- `list_project_versions` - List project versions +- `search_projects` - Search projects by name +- `get_project_vulnerabilities` - Get security vulnerabilities +- `list_project_components` - List project components + +See [MCP_INTEGRATION.md](MCP_INTEGRATION.md) for detailed documentation. + # Version History Including a version history on a go-forward basis. +## v1.1.4 (Upcoming) + +Added Model Context Protocol (MCP) integration: +- New `blackduck --mcp` CLI command to start MCP server +- Support for AI assistants via MCP tools +- Optional FastMCP dependency via `pip install blackduck[mcp]` +- Six core MCP tools for project and vulnerability management + ## v1.1.0 Retries will be attempted for all HTTP verbs, not just GET. diff --git a/blackduck/Authentication.py b/blackduck/Authentication.py index c0f1f7ec..e6b22ea1 100644 --- a/blackduck/Authentication.py +++ b/blackduck/Authentication.py @@ -56,7 +56,7 @@ def authenticate(self): logger.warning("ssl verification disabled, connection insecure. do NOT use verify=False in production!") response = self.session.post( - url="/api/tokens/authenticate", + url="api/tokens/authenticate", auth=NoAuth(), # temporarily strip authentication to avoid infinite recursion headers={"Authorization": f"token {self.access_token}"} ) diff --git a/blackduck/Core.py b/blackduck/Core.py index abed0867..53af5eae 100644 --- a/blackduck/Core.py +++ b/blackduck/Core.py @@ -146,6 +146,17 @@ def execute_put(self, url, data, custom_headers={}): response = requests.put(url, headers=headers, data=json_data, verify = not self.config['insecure']) return response +def execute_patch(self, url, data, custom_headers={}) -> requests.Response: + ''' + Work-around to add missing execute_patch() + ''' + json_data = self._validated_json_data(data) + headers = self.get_headers() + headers["Content-Type"] = "application/json" + headers.update(custom_headers) + response = requests.patch(url, headers=headers, data=json_data, verify=not self.config['insecure']) + return response + def _create(self, url, json_body): response = self.execute_post(url, json_body) # v4+ returns the newly created location in the response headers diff --git a/blackduck/HubRestApi.py b/blackduck/HubRestApi.py index d3ddb43e..9fcd0259 100755 --- a/blackduck/HubRestApi.py +++ b/blackduck/HubRestApi.py @@ -68,7 +68,7 @@ class HubInstance(object): _create,_get_hub_rest_api_version_info,_get_major_version,_get_parameter_string,_validated_json_data, execute_delete,execute_get,execute_post,execute_put,get_api_version,get_apibase,get_auth_token,get_headers, get_limit_paramstring,get_link,get_matched_components,get_tags_url,get_urlbase,read_config,write_config, - _check_version_compatibility + _check_version_compatibility,execute_patch ) from .Roles import ( _get_role_url, assign_role_given_role_url, assign_role_to_user_or_group, diff --git a/blackduck/Reporting.py b/blackduck/Reporting.py index 70428191..7735a604 100644 --- a/blackduck/Reporting.py +++ b/blackduck/Reporting.py @@ -25,7 +25,7 @@ def create_version_reports(self, version, report_list, format="CSV"): return self.execute_post(version_reports_url, post_data) valid_notices_formats = ["TEXT", "JSON"] -def create_version_notices_report(self, version, format="TEXT", include_copyright_info=True): +def create_version_notices_report(self, version, format="TEXT", include_copyright_info=True, include_license_info=True): assert format in valid_notices_formats, "Format must be one of {}".format(valid_notices_formats) post_data = { @@ -35,6 +35,8 @@ def create_version_notices_report(self, version, format="TEXT", include_copyrigh } if include_copyright_info: post_data.update({'categories': ["COPYRIGHT_TEXT"] }) + if include_license_info: + post_data.update({'categories': ["COPYRIGHT_TEXT","LICENSE_DATA","LICENSE_TEXT"] }) notices_report_url = self.get_link(version, 'licenseReports') return self.execute_post(notices_report_url, post_data) diff --git a/blackduck/Vulnerabilities.py b/blackduck/Vulnerabilities.py index fe3d665c..b9f4929d 100644 --- a/blackduck/Vulnerabilities.py +++ b/blackduck/Vulnerabilities.py @@ -28,6 +28,7 @@ def get_vulnerable_bom_components(self, version_obj, limit=9999): param_string = self._get_parameter_string({'limit': limit}) url = "{}{}".format(url, param_string) response = self.execute_get(url, custom_headers=custom_headers) + response.raise_for_status() return response.json() # TODO: Remove or refactor this diff --git a/blackduck/__main__.py b/blackduck/__main__.py new file mode 100644 index 00000000..fdd2b6a3 --- /dev/null +++ b/blackduck/__main__.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +""" +BlackDuck CLI entry point +""" + +import sys +import argparse +import os +import logging + +def main(): + """Main CLI entry point for BlackDuck""" + parser = argparse.ArgumentParser( + prog='blackduck', + description='BlackDuck Hub REST API CLI' + ) + + parser.add_argument( + '--mcp', + action='store_true', + help='Launch MCP (Model Context Protocol) server' + ) + + parser.add_argument( + '--version', + action='version', + version=f'blackduck {get_version()}' + ) + + args = parser.parse_args() + + if args.mcp: + launch_mcp_server() + else: + show_usage() + +def get_version(): + """Get package version""" + try: + from .__version__ import __version__ + return __version__ + except ImportError: + return 'unknown' + +def show_usage(): + """Show basic usage information""" + print("BlackDuck Hub REST API CLI") + print() + print("Usage:") + print(" blackduck --mcp Launch MCP server") + print(" blackduck --version Show version") + print() + print("For MCP server usage:") + print(" Set environment variables:") + print(" BLACKDUCK_URL") + print(" BLACKDUCK_TOKEN") + +def launch_mcp_server(): + """Launch the MCP server""" + try: + from .mcp_server import run_mcp_server + run_mcp_server() + except ImportError as e: + print(f"Error: MCP server dependencies not available: {e}", file=sys.stderr) + print("Install with: pip install fastmcp", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error launching MCP server: {e}", file=sys.stderr) + sys.exit(1) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/blackduck/__version__.py b/blackduck/__version__.py index 4eaaf1f8..4d580b10 100644 --- a/blackduck/__version__.py +++ b/blackduck/__version__.py @@ -1,3 +1,3 @@ -VERSION = (1, 1, 0) +VERSION = (1, 1, 3) __version__ = '.'.join(map(str, VERSION)) diff --git a/blackduck/mcp_server.py b/blackduck/mcp_server.py new file mode 100644 index 00000000..48ba55f1 --- /dev/null +++ b/blackduck/mcp_server.py @@ -0,0 +1,399 @@ +""" +BlackDuck MCP Server + +Provides Model Context Protocol interface for BlackDuck Hub REST API +""" + +import os +import sys +import logging +from typing import List, Dict, Any, Optional +import json + +# Configure logging for MCP server +logging.basicConfig(level=logging.WARNING) +logger = logging.getLogger(__name__) + +try: + from fastmcp import FastMCP +except ImportError: + raise ImportError( + "FastMCP not available. Install with: pip install fastmcp" + ) + +from .Client import Client +from .Utils import safe_get + + +class BlackDuckMCPServer: + """BlackDuck MCP Server implementation""" + + def __init__(self): + self.mcp = FastMCP("BlackDuck Hub") + self.client = None + self._setup_client() + self._register_tools() + + def _setup_client(self): + """Initialize BlackDuck client from environment variables""" + base_url = os.environ.get('BLACKDUCK_URL') + token = os.environ.get('BLACKDUCK_TOKEN') + + if not base_url or not token: + raise ValueError( + "Missing required environment variables:\n" + " BLACKDUCK_URL and BLACKDUCK_TOKEN" + ) + + self.client = Client( + base_url=base_url, + token=token, + verify=True, # Default to secure + timeout=30.0, + retries=3 + ) + + logger.info(f"BlackDuck client initialized for {base_url}") + + def _register_tools(self): + """Register MCP tools""" + + @self.mcp.tool + def list_projects(limit: Optional[int] = 50) -> List[Dict[str, Any]]: + """List BlackDuck projects + + Args: + limit: Maximum number of projects to return (default: 50) + + Returns: + List of project dictionaries with name, description, and metadata + """ + try: + projects = self.client.get_resource('projects') + result = [] + + for i, project in enumerate(projects): + if limit and i >= limit: + break + + result.append({ + 'name': project.get('name'), + 'description': project.get('description', ''), + 'projectOwner': safe_get(project, 'projectOwner'), + 'createdAt': project.get('createdAt'), + 'updatedAt': project.get('updatedAt'), + '_meta': { + 'href': project.get('_meta', {}).get('href') + } + }) + + return result + + except Exception as e: + logger.error(f"Error listing projects: {e}") + raise + + @self.mcp.tool + def get_project_details(project_name: str) -> Optional[Dict[str, Any]]: + """Get detailed information about a specific project + + Args: + project_name: Name of the project to retrieve + + Returns: + Project details dictionary or None if not found + """ + try: + params = {'q': [f"name:{project_name}"]} + projects = list(self.client.get_resource('projects', params=params)) + + # Find exact match (case-insensitive) + project = None + for p in projects: + if p['name'].lower() == project_name.lower(): + project = p + break + + if not project: + return None + + return { + 'name': project.get('name'), + 'description': project.get('description', ''), + 'projectOwner': safe_get(project, 'projectOwner'), + 'createdAt': project.get('createdAt'), + 'updatedAt': project.get('updatedAt'), + 'projectLevelAdjustments': project.get('projectLevelAdjustments', False), + 'cloneCategories': project.get('cloneCategories', []), + '_meta': project.get('_meta', {}) + } + + except Exception as e: + logger.error(f"Error getting project details: {e}") + raise + + @self.mcp.tool + def list_project_versions(project_name: str, limit: Optional[int] = 20) -> List[Dict[str, Any]]: + """List versions for a specific project + + Args: + project_name: Name of the project + limit: Maximum number of versions to return (default: 20) + + Returns: + List of version dictionaries + """ + try: + # First find the project + params = {'q': [f"name:{project_name}"]} + projects = list(self.client.get_resource('projects', params=params)) + + project = None + for p in projects: + if p['name'].lower() == project_name.lower(): + project = p + break + + if not project: + return [] + + # Get versions for the project + versions = self.client.get_resource('versions', project) + result = [] + + for i, version in enumerate(versions): + if limit and i >= limit: + break + + result.append({ + 'versionName': version.get('versionName'), + 'nickname': version.get('nickname'), + 'phase': version.get('phase'), + 'distribution': version.get('distribution'), + 'createdAt': version.get('createdAt'), + 'settingUpdatedAt': version.get('settingUpdatedAt'), + '_meta': { + 'href': version.get('_meta', {}).get('href') + } + }) + + return result + + except Exception as e: + logger.error(f"Error listing project versions: {e}") + raise + + @self.mcp.tool + def search_projects(query: str, limit: Optional[int] = 25) -> List[Dict[str, Any]]: + """Search for projects by name or description + + Args: + query: Search query string + limit: Maximum number of results to return (default: 25) + + Returns: + List of matching project dictionaries + """ + try: + params = {'q': [f"name:{query}"]} + projects = self.client.get_resource('projects', params=params) + result = [] + + for i, project in enumerate(projects): + if limit and i >= limit: + break + + # Simple relevance scoring + name = project.get('name', '').lower() + description = project.get('description', '').lower() + query_lower = query.lower() + + relevance = 0 + if query_lower in name: + relevance += 2 + if query_lower in description: + relevance += 1 + + result.append({ + 'name': project.get('name'), + 'description': project.get('description', ''), + 'relevance': relevance, + 'createdAt': project.get('createdAt'), + '_meta': { + 'href': project.get('_meta', {}).get('href') + } + }) + + # Sort by relevance + result.sort(key=lambda x: x['relevance'], reverse=True) + return result + + except Exception as e: + logger.error(f"Error searching projects: {e}") + raise + + @self.mcp.tool + def get_project_vulnerabilities(project_name: str, version_name: Optional[str] = None, limit: Optional[int] = 50) -> List[Dict[str, Any]]: + """Get vulnerabilities for a project version + + Args: + project_name: Name of the project + version_name: Name of the version (if None, uses latest) + limit: Maximum number of vulnerabilities to return (default: 50) + + Returns: + List of vulnerability dictionaries + """ + try: + # Find project + params = {'q': [f"name:{project_name}"]} + projects = list(self.client.get_resource('projects', params=params)) + + project = None + for p in projects: + if p['name'].lower() == project_name.lower(): + project = p + break + + if not project: + return [] + + # Find version + versions = list(self.client.get_resource('versions', project)) + if not versions: + return [] + + version = None + if version_name: + for v in versions: + if v['versionName'].lower() == version_name.lower(): + version = v + break + else: + # Use first (most recent) version + version = versions[0] + + if not version: + return [] + + # Get vulnerabilities + try: + vulnerabilities = self.client.get_resource('vulnerable-components', version) + result = [] + + for i, vuln in enumerate(vulnerabilities): + if limit and i >= limit: + break + + result.append({ + 'componentName': vuln.get('componentName'), + 'componentVersionName': vuln.get('componentVersionName'), + 'vulnerabilityName': vuln.get('vulnerabilityName'), + 'severity': vuln.get('severity'), + 'baseScore': vuln.get('baseScore'), + 'overallScore': vuln.get('overallScore'), + 'remediationStatus': vuln.get('remediationStatus'), + 'description': vuln.get('description', ''), + 'publishedDate': vuln.get('publishedDate'), + 'updatedDate': vuln.get('updatedDate') + }) + + return result + + except Exception: + # Fallback: try to get components instead + components = self.client.get_resource('components', version) + return [{'info': 'Use list_project_components for component information'}] + + except Exception as e: + logger.error(f"Error getting vulnerabilities: {e}") + raise + + @self.mcp.tool + def list_project_components(project_name: str, version_name: Optional[str] = None, limit: Optional[int] = 50) -> List[Dict[str, Any]]: + """List components in a project version + + Args: + project_name: Name of the project + version_name: Name of the version (if None, uses latest) + limit: Maximum number of components to return (default: 50) + + Returns: + List of component dictionaries + """ + try: + # Find project + params = {'q': [f"name:{project_name}"]} + projects = list(self.client.get_resource('projects', params=params)) + + project = None + for p in projects: + if p['name'].lower() == project_name.lower(): + project = p + break + + if not project: + return [] + + # Find version + versions = list(self.client.get_resource('versions', project)) + if not versions: + return [] + + version = None + if version_name: + for v in versions: + if v['versionName'].lower() == version_name.lower(): + version = v + break + else: + version = versions[0] + + if not version: + return [] + + # Get components + components = self.client.get_resource('components', version) + result = [] + + for i, component in enumerate(components): + if limit and i >= limit: + break + + licenses = component.get('licenses', []) + license_display = licenses[0].get('licenseDisplay', 'Unknown') if licenses else 'Unknown' + + result.append({ + 'componentName': component.get('componentName'), + 'componentVersionName': component.get('componentVersionName'), + 'matchTypes': component.get('matchTypes', []), + 'usages': component.get('usages', []), + 'licenseDisplay': license_display, + 'policyStatus': component.get('policyStatus'), + 'securityRiskProfile': component.get('securityRiskProfile'), + 'activityData': component.get('activityData') + }) + + return result + + except Exception as e: + logger.error(f"Error listing components: {e}") + raise + + def run(self): + """Run the MCP server""" + self.mcp.run() + + +def run_mcp_server(): + """Entry point for running the MCP server""" + try: + server = BlackDuckMCPServer() + server.run() + except Exception as e: + logger.error(f"Failed to start MCP server: {e}") + sys.exit(1) + + +if __name__ == '__main__': + run_mcp_server() \ No newline at end of file diff --git a/examples/client/batch_generate_sbom.py b/examples/client/batch_generate_sbom.py index e27374d1..d8a39a84 100644 --- a/examples/client/batch_generate_sbom.py +++ b/examples/client/batch_generate_sbom.py @@ -262,7 +262,7 @@ def sanitize_filename(filename): def parse_command_args(): - parser = argparse.ArgumentParser("Generate and download reports for projets in a spreadsheet") + parser = argparse.ArgumentParser("Generate and download reports for projects in a spreadsheet") parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") parser.add_argument("-t", "--token-file", required=True, help="File containing access token") parser.add_argument("-i", "--input-file", required=True, help="Project Name") diff --git a/examples/client/consolidated_file_report.py b/examples/client/consolidated_file_report.py new file mode 100755 index 00000000..f014799d --- /dev/null +++ b/examples/client/consolidated_file_report.py @@ -0,0 +1,823 @@ +''' +Created on August 30, 2023 + +@author: mkoishi + +Generate reports which consolidates information on BOM components, versions with license information, BOM files with +license and copyright information from KB and license search (discoveries - file licenses or file copyrights), +and BlackDuck unmatched files in the target source code. + +Copyright (C) 2023 Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +''' + +import argparse +import logging +import sys +import os +import re +import time +import subprocess +import json +import traceback +import copy +import ijson +from json2html import * +from blackduck import Client +from zipfile import ZipFile + +program_description = \ +'''This script collects BlackDuck reports for version details and discoveries and generates new reports. +The newly generated reports are a report of BOM component-versions with license information, a report of BOM files that consolidates licenses and copyright texts information from KB and discoveries (e.g., file licenses, file copyrights), and a report of BlackDuck unmatched files. +The generated reports are found in a folder named "blackduck_consolidated_file_report" +Synopsys-Detect is executed from the script unless skip-detect option is chosen. Synopsys-Detect runs in synchronous mode and please consider adjusting the "detect.timeout" detect parameter if completion of Synopsys-Detect is estimated to take longer. + +Config file: +API Token, hub URL and two more options need to be placed in the .restconfig.json file that must be placed in the same folder where this script resides. +For more information, please find the instructions in the linked contents at https://community.synopsys.com/s/article/How-to-use-the-hub-rest-api-python-for-Black-Duck + +Pre-requisites: +1) python (>=3.5) and pip are installed. +2) Install PyPI modules "blackduck", "ijson" and "json2html. + +Examples: +1) If Synopsys-Detect is wanted to execute prior to the HTML report generation and the report should include file-copyright texts, then +python3 ./consolidated_file_report.py \ +-f html -cl 2 -rr 100 \ +-dp detect.blackduck.signature.scanner.snippet.matching=SNIPPET_MATCHING \ +detect.blackduck.signature.scanner.license.search=true \ +detect.blackduck.signature.scanner.copyright.search=true \ +detect.source.path= \ +detect.timeout=1800 \ +blackduck.trust.cert=true + +Note: blackduck.trust.cert=true is not recommended in the production environment. + +2) If execution of Synopsys-Detect is wanted to bypass and no copyright texts are needed in the file report, then +python3 ./consolidated_file_report.py \ +-f html -sd -rr 100 -dp detect.source.path= + +Note: Please provide -dp detect.source.path parameter. If otherwise, the script has no clue for the target source folder. +''' + +# Synopsys Detect parameters +DOWNLOAD_DETECT = ["curl", "-s", "-L", "https://detect.synopsys.com/detect8.sh"] +BLACKDUCK_URL = "--blackduck.url=" +BLACKDUCK_TOKEN = "--blackduck.api.token=" +BLACKDUCK_PROJECT = "--detect.project.name=" +BLACKDUCK_VERSION = "--detect.project.version.name=" +BLACKDUCK_WAIT = "--detect.wait.for.results=true" +ENV_DETECT_VERSION = "DETECT_LATEST_RELEASE_VERSION" +# BD report general +BLACKDUCK_REPORT_MEDIATYPE = "application/vnd.blackducksoftware.report-4+json" +blackduck_report_download_api = "/api/projects/{projectId}/versions/{projectVersionId}/reports/{reportId}/download" +blackduck_link_component_ui_api = "/api/projects/{projectId}/versions/{projectVersionId}/components" +blackduck_link_snippet_ui_api = "/api/projects/{projectId}/versions/{projectVersionId}/source-trees" +# BD version details report +blackduck_create_version_report_api = "/api/versions/{projectVersionId}/reports" +blackduck_version_report_filename = "./blackduck_version_report_for_{projectVersionId}.zip" +# BD discoveries report +blackduck_create_discoveries_report_api = "/api/versions/{projectVersionId}/license-reports" +blackduck_discoveries_report_filename = "./blackduck_discoveries_report_for_{projectVersionId}.zip" +# Consolidated report +BLACKDUCK_VERSION_MEDIATYPE = "application/vnd.blackducksoftware.status-4+json" +BLACKDUCK_VERSION_API = "/api/current-version" +BLACKDUCK_SOURCE_PATH = "detect.source.path" +REPORT_DIR = "./blackduck_consolidated_file_report" +REPORT_HEADER = "/header_report" +REPORT_OS_FILE = "/os_file_report" +REPORT_COMPONENT_BOM = "/component_bom_report" +REPORT_FILE_BOM = "/file_bom_report" +REPORT_DISCOVERY = "/discovery" +BLACKDUCK_SNIPPET_FILTER = "?filter=bomMatchType%3Asnippet&offset=0&limit=100" +# Retries to wait for BD report creation. RETRY_LIMIT can be overwritten by the script parameter. +RETRY_LIMIT = 30 +RETRY_TIMER = 30 +# Reports +report_content = { + 'title' : 'Black Duck Consolidated File Report', + 'configurationSettings':{ + 'detectParameters': [], + 'scanDateTime': '', + 'blackDuckVersion': '', + 'linkToBlackDuckProjectVersionInUI': '', + 'linkToBlackDuckSnippetMatchInUI': '' + }, + 'fileInventory': { + 'linkToUnmatchedOsFileData': "", + 'linkToBomComponentEntries': "", + 'linkToBomFileEntries': "", + } +} +report_os_file = { + 'unmatchedOsFileEntries': { + 'description': 'This list includes folders and files which belong to the target source directory and are not matched by BlackDuck', + 'unmatched': [] + } +} +report_component_bom = { + 'bomComponentEntries': { + 'description': 'This list includes BOM component information which is extracted from BlackDuck project version report.', + 'bomComponents': [] + } +} +report_file_bom = { + 'bomFileEntries': { + 'description': 'This list includes BOM file information which is extracted from BlackDuck project version report and Discovery report.', + 'bomFiles': [], + 'unmatchedFileDiscoveries': [] + } +} + +def log_config(debug): + if debug: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.DEBUG) + else: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.INFO) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("blackduck").setLevel(logging.WARNING) + +def parse_parameter(): + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("project", + metavar="project", + type=str, + help="Provide the BlackDuck project name.") + parser.add_argument("version", + metavar="version", + type=str, + help="Provide the BlackDuck project version name.") + parser.add_argument("-f", + "--report_format", + metavar="", + type=str, + default="json", + help="Specify the report format. Currently either JSON or HTML is supported. Default is JSON.") + parser.add_argument("-cl", + "--copyright_level", + metavar="", + type=int, + default=0, + help="Specify the included copyright text level. Level 0 (default value) is no copyright texts included, 1 is only copyright texts from KB included, 2 is copyright texts from KB and discoveries included.") + parser.add_argument("-sd", + "--skip_detect", + action='store_true', + help="Set if execution of Synopsys-Detect is wanted to bypass.") + parser.add_argument("-dv", + "--detect_version", + metavar="", + type=str, + default="latest", + help="Specify the Synopsys Detect version to download and run. If not set, the latest version will run.") + parser.add_argument("-rr", + "--report_retries", + metavar="", + type=int, + default=RETRY_LIMIT, + help="Retries for receiving the generated BlackDuck report. Timeout timer is hard-coded 30 sec. Generating a copyright report tends to take longer.") + parser.add_argument("-t", + "--timeout", + metavar="", + type=int, + default=15, + help="Timeout for REST-API. Some API may take longer than the default 15 seconds") + parser.add_argument("-r", + "--retries", + metavar="", + type=int, + default=3, + help="Retries for REST-API. Some API may need more retries than the default 3 times") + parser.add_argument("-dp", + "--detect_parameters", + metavar="", + type=str, + nargs="*", + default="", + help="List Synopsys Detect parameters with whitespace separators and without '--'. Example: -dp detect.blackduck.signature.scanner.snippet.matching=SNIPPET_MATCHING detect.blackduck.signature.scanner.license.search=true ") + return parser.parse_args() + +def run_detect(project, version, bd_url, bd_token, detect_version, bd_params=None): + """ Download and run Synopsys Detect. """ + # TODO: Consider to change to async and pall completion of the scan because that maybe more robust against network errors. + detect_params = [ + BLACKDUCK_URL + bd_url, + BLACKDUCK_TOKEN + bd_token, + BLACKDUCK_PROJECT + project, + BLACKDUCK_VERSION + version, + BLACKDUCK_WAIT] + if bd_params is not None: + for param in bd_params: + detect_params.append(f"--{param}") + + # Download the designated detect version of synopsys-detect + if detect_version != "latest": + major_version = detect_version.split(".", 1)[0] + for i, element in enumerate(DOWNLOAD_DETECT): + if element == "https://detect.synopsys.com/detect8.sh": + DOWNLOAD_DETECT[i] = element.replace("detect8", f"detect{major_version}") + break + my_env = os.environ.copy() + my_env[ENV_DETECT_VERSION] = detect_version + with open("synopsys-detect.sh", "w") as detect: + logging.info(f"Synopsys Detect version {detect_version} is being downloaded now!") + subprocess.run(DOWNLOAD_DETECT, stdout=detect) + + detect_command = ["bash", f"{detect.name}"] + detect_params + logging.info(f"synopsys-detect synchronous scan is in execution now!") + if detect_version != "latest": + results = subprocess.run(detect_command, capture_output=True, text=True, env=my_env) + else: + results = subprocess.run(detect_command, capture_output=True, text=True) + + print(results.stdout) + print(results.stderr) + results.check_returncode() + return + +def get_bd_project_data(hub_client, project_name, version_name): + """ Get and return project ID, version ID and codelocations. """ + project_id = "" + for project in hub_client.get_resource("projects"): + if project['name'] == project_name: + project_id = (project['_meta']['href']).split("projects/", 1)[1] + break + if project_id == "": + sys.exit(f"No project for {project_name} was found!") + version_id = codelocations = "" + for version in hub_client.get_resource("versions", project): + if version['versionName'] == version_name: + version_id = (version['_meta']['href']).split("versions/", 1)[1] + for link in version['_meta']['links']: + if link['rel'] == "codelocations": + codelocations = link['href'] + break + break + if version_id == "": + sys.exit(f"No project version for {version_name} was found!") + if codelocations == "": + sys.exit(f"No codelocations for {project_name} {version_name} found ") + + return project_id, version_id, codelocations + +def report_create(hub_client, url, body): + """ + Request BlackDuck to create report. Requested report is included in the request payload. + """ + res = hub_client.session.post(url, headers={'Content-Type': BLACKDUCK_REPORT_MEDIATYPE}, json=body) + if res.status_code != 201: + sys.exit(f"BlackDuck report creation failed with status {res.status_code}!") + # return report_url + return res.headers['Location'] + +def report_download(hub_client, report_url, project_id, version_id, retry_count): + """ + Download the generated report after the report completion. We will retry until reaching the retry-limit. + """ + retries = retry_count + while retries: + res = hub_client.session.get(report_url, headers={'Accept': BLACKDUCK_REPORT_MEDIATYPE}) + if res.status_code == 200 and (json.loads(res.content))['status'] == "COMPLETED": + report_id = report_url.split("reports/", 1)[1] + download_url = (((blackduck_report_download_api.replace("{projectId}", project_id)) + .replace("{projectVersionId}", version_id)) + .replace("{reportId}", report_id)) + res = hub_client.session.get(download_url, + headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + if res.status_code != 200: + sys.exit(f"BlackDuck report download failed with status {res.status_code} for {download_url}!") + return res.content + elif res.status_code != 200: + sys.exit(f"BlackDuck report creation not completed successfully with status {res.status_code}") + else: + logging.info(f"Waiting for the report generation for {report_url} with the remaining retries {retries} times.") + retries -= 1 + time.sleep(RETRY_TIMER) + sys.exit(f"BlackDuck report for {report_url} was not generated after retries {RETRY_TIMER} sec * {retry_count} times!") + +def get_version_detail_report(hub_client, project_id, version_id, retries): + """ Create and get BOM component and BOM source file report in json. """ + create_version_url = blackduck_create_version_report_api.replace("{projectVersionId}", version_id) + body = { + 'reportFormat' : 'JSON', + 'locale' : 'en_US', + 'versionId' : f'{version_id}', + 'categories' : [ 'COMPONENTS', 'FILES' ] + } + report_url = report_create(hub_client, create_version_url, body) + # Zipped report content is received and write the content to a local zip file + content = report_download(hub_client, report_url, project_id, version_id, retries) + output_file = blackduck_version_report_filename.replace("{projectVersionId}", version_id) + with open(output_file, "wb") as f: + f.write(content) + return output_file + +def get_discovery_report(hub_client, project_id, version_id, retries, copyright): + """ Create and get discovery report for licenses and copyrights in json. """ + create_discoveries_url = blackduck_create_discoveries_report_api.replace("{projectVersionId}", version_id) + body = { + 'reportFormat' : 'JSON', + 'locale' : 'en_US', + 'versionId' : f'{version_id}', + 'categories' : [ 'FILE_LICENSE_DATA', 'DEEP_LICENSE_DATA', 'UNMATCHED_FILE_DISCOVERIES' ] + } + if copyright == 1: + body['categories'].append('COPYRIGHT_TEXT') + elif copyright ==2: + body['categories'].extend(['COPYRIGHT_TEXT', 'FILE_COPYRIGHT_TEXT']) + report_url = report_create(hub_client, create_discoveries_url, body) + content = report_download(hub_client, report_url, project_id, version_id, retries) + output_file = blackduck_discoveries_report_filename.replace("{projectVersionId}", version_id) + with open(output_file, "wb") as f: + f.write(content) + return output_file + +def get_folder_size(path="."): + """ + Calculate the size of the given folder by summing up belonging files. + Remarks: Calculated folder size maybe shows different figure from OS command line because + OS commands may take the file system's chunk into consideration. + """ + total_size = 0 + with os.scandir(path) as it: + for entry in it: + if entry.is_file(): + size = entry.stat().st_size + total_size = total_size + size + elif entry.is_dir(): + size = get_folder_size(entry.path) + total_size = total_size + size + return total_size + +def get_os_path_for_unmatched(parent_dir, matched_paths): + """ + Traverse within the provided directory and yield OS folder or file data if it is not a matched path by BlackDuck. + """ + os_path_data = { + 'path': "", + 'sizeInBytes': 0 + } + os_path_stats = { + 'matched_folders': 0, + 'matched_files': 0, + 'unmatched_folders': 0, + 'unmatched_files': 0, + 'total_folders': 0, + 'total_files': 0 + } + + log_onerror = lambda err: logging.error(f"An error reported during OS folder and file traverse. OS file report may be uncompleted.{str(err)}") + for path, folder_names, file_names in os.walk(parent_dir, onerror=log_onerror): + os_path_stats['total_folders'] = os_path_stats['total_folders'] + len(folder_names) + os_path_stats['total_files'] = os_path_stats['total_files'] + len(file_names) + + for folder_name in folder_names: + file_data = copy.deepcopy(os_path_data) + try: + abs_path = os.path.join(path, folder_name) + # Removal of parent path and add a slash for comparison with BlackDuck reported path + rel_path = abs_path.replace(f"{parent_dir}/", "") + "/" + if rel_path in matched_paths: + os_path_stats['matched_folders'] = os_path_stats['matched_folders'] + 1 + continue + file_data['path'] = rel_path + file_data['sizeInBytes'] = get_folder_size(abs_path) + os_path_stats['unmatched_folders'] = os_path_stats['unmatched_folders'] + 1 + # Keep the following log if something is wrong with the size + # logging.debug(f"filepath {file_data['path']} with size {file_data['sizeInBytes']} bytes") + yield file_data + except Exception as err: + logging.warning(f"An exception raised during OS folder report. {str(err)}") + continue + + for file_name in file_names: + file_data = copy.deepcopy(os_path_data) + try: + abs_path = os.path.join(path, file_name) + # Removal of parent path for comparison with BlackDuck reported file path + rel_path = abs_path.replace(f"{parent_dir}/", "") + if rel_path in matched_paths: + os_path_stats['matched_files'] = os_path_stats['matched_files'] + 1 + continue + file_data['path'] = rel_path + file_data['sizeInBytes'] = os.path.getsize(abs_path) + os_path_stats['unmatched_files'] = os_path_stats['unmatched_files'] + 1 + #logging.debug(f"filepath {file_data['path']} with size {file_data['sizeInBytes']} bytes") + yield file_data + except Exception as err: + logging.warning(f"An exception raised during OS file report. {str(err)}") + continue + + logging.debug(f"Matched folders: {os_path_stats['matched_folders']}, Matched files: {os_path_stats['matched_files']}, " + f"Unmatched folders: {os_path_stats['unmatched_folders']}, Unmatched files: {os_path_stats['unmatched_files']}, " + f"Total folders under the parent path: {os_path_stats['total_folders']}, " + f"Total files under the parent path: {os_path_stats['total_files']}, " + f"Number of matched paths by BlackDuck: {len(matched_paths)}") + return + +def pull_component_bom(component_bom): + """ + Extract BOM component data from BD project version report. + Remarks: Component BOM data includes components for SNIPPET matching which can outnumber the number of the components + in BlackDuck project version BOM UI. + """ + component_info = { + 'componentName': '', + 'componentVersionNames': [], + 'matchTypes': [], + 'licenses': [] + } + license_info = { + 'licenseType': '', + 'name': '', + 'licenseFamily': '', + 'licenses': [], + 'licenseDisplay': '' + } + license_nested = { + 'name': '', + 'licenseFamily': '', + 'licenses': [], + 'licenseDisplay': '' + } + + component_bom_data = copy.deepcopy(component_info) + component_bom_data['componentName'] = component_bom['producerProject']['name'] \ + if 'producerProject' in component_bom.keys() and 'name' in component_bom['producerProject'].keys() else "" + if 'producerReleases' in component_bom.keys(): + for release in component_bom['producerReleases']: + component_bom_data['componentVersionNames'].append(release['version']) + if 'matchTypes' in component_bom.keys(): + for match in component_bom['matchTypes']: + component_bom_data['matchTypes'].append(match) + if 'licenses' in component_bom.keys(): + for license in component_bom['licenses']: + license_data = copy.deepcopy(license_info) + license_data['licenseType'] = license['licenseType'] if 'licenseType' in license.keys() else "" + license_data['name'] = license['name'] if 'name' in license.keys() else "" + license_data['licenseFamily'] = license['codeSharing'] if 'codeSharing' in license.keys() else "" + if 'licenses' in license.keys(): + for in_license in license['licenses']: + nested_license_data = copy.deepcopy(license_nested) + nested_license_data['name'] = in_license['name'] if 'name' in in_license.keys() else "" + nested_license_data['licenseFamily'] = in_license['codeSharing'] if 'codeSharing' in in_license.keys() else "" + nested_license_data['licenseDisplay'] = in_license['licenseDisplay'] if 'licenseDisplay' in in_license.keys() else "" + nested_license_data['licenses'] = in_license['licenses'] if 'licenses' in in_license.keys() else [] + license_data['licenses'].append(nested_license_data) + license_data['licenseDisplay']= license['licenseDisplay'] if 'licenseDisplay' in license.keys() else "" + component_bom_data['licenses'].append(license_data) + return component_bom_data + +def pull_file_bom(file_bom): + """ + Extract BOM file data from BD project version report. + Remarks: If reported path is ended with '/', then that is folder path. If not, that is file path. + """ + + file_info = { + 'path' : '', + 'archiveContext' : '', + 'projectName' : '', + 'projectVersion' : '', + 'channelReleaseExternalNamespace' : '', + 'channelReleaseExternalId' : '', + 'matchType' : '', + 'snippetReviewStatus' : '', + 'licenses' : [], + 'copyrights' : { + 'copyrightTexts' : [], + 'fileCopyrightTexts' : [] + } + } + + file_bom_data = copy.deepcopy(file_info) + file_bom_data['path'] = file_bom['path'] if 'path' in file_bom.keys() else "" + file_bom_data['archiveContext'] = file_bom['archiveContext'] if 'archiveContext' in file_bom.keys() else "" + file_bom_data['projectName'] = file_bom['projectName'] if 'projectName' in file_bom.keys() else "" + file_bom_data['projectVersion'] = file_bom['version'] if 'version' in file_bom.keys() else "" + file_bom_data['matchType'] = file_bom['matchType'] if 'matchType' in file_bom.keys() else "" + if 'matchType' in file_bom.keys() and file_bom['matchType'] == "SNIPPET": + file_bom_data['snippetReviewStatus'] = file_bom['snippetReviewStatus'] + file_bom_data['channelReleaseExternalNamespace'] = file_bom['channelReleaseExternalNamespace'] \ + if 'channelReleaseExternalNamespace' in file_bom.keys() else "" + file_bom_data['channelReleaseExternalId'] = file_bom['channelReleaseExternalId'] \ + if 'channelReleaseExternalId' in file_bom.keys() else "" + return file_bom_data + +def pull_discovery_licenses(comp_license): + """ + Extract component licenses from discoveries. + """ + comp_license_info = { + 'projectName' : '', + 'versionName' : '', + 'licenses' : [] + } + license_info = { + 'name' : '', + 'sources' : [] + } + comp_license_data = copy.deepcopy(comp_license_info) + comp_license_data['projectName'] = comp_license['component']['projectName'] \ + if 'component' in comp_license.keys() and 'projectName' in comp_license['component'].keys() else "" + comp_license_data['versionName'] = comp_license['component']['versionName'] \ + if 'component' in comp_license.keys() and 'versionName' in comp_license['component'].keys() else "" + if 'licenses' in comp_license.keys(): + for license in comp_license['licenses']: + license_data = copy.deepcopy(license_info) + license_data['name'] = license['name'] if 'name' in license.keys() else "" + if 'sources' in license.keys(): + for source in license['sources']: + license_data['sources'].append(source) + comp_license_data['licenses'].append(license_data) + return comp_license_data + +def pull_discovery_copyrights(comp_copyright): + """ + Extract component copyrights from discoveries. + """ + comp_copyright_info = { + 'originFullName' : '', + 'copyrights' : { + 'copyrightTexts' : [], + 'fileCopyrightTexts' : [] + } + } + comp_copyright_data = copy.deepcopy(comp_copyright_info) + comp_copyright_data['originFullName'] = comp_copyright['originFullName'] if 'originFullName' in comp_copyright.keys() else "" + if 'copyrightTexts' in comp_copyright.keys(): + for copyright in comp_copyright['copyrightTexts']: + comp_copyright_data['copyrights']['copyrightTexts'].append(copyright) + if 'fileCopyrightTexts' in comp_copyright.keys(): + for file_copyright in comp_copyright['fileCopyrightTexts']: + comp_copyright_data['copyrights']['fileCopyrightTexts'].append(file_copyright) + return comp_copyright_data + +def pull_discovery_unmatched(comp_unmatched): + """ + Extract unmatched files from discoveries. + """ + comp_unmatched_info = { + 'fileNames' : [], + 'resourceName' : '', + 'matchType' : '', + 'matchTypeLabel' : '' + } + file_name_info = { + 'path' : '', + 'archiveContext' : '', + 'compositePathContext' : '', + 'fileName' : '' + } + + comp_unmatched_data = copy.deepcopy(comp_unmatched_info) + if 'fileNames' in comp_unmatched.keys(): + for file_name in comp_unmatched['fileNames']: + file_name_data = copy.deepcopy(file_name_info) + file_name_data['path'] = file_name['path'] if 'path' in file_name.keys() else "" + file_name_data['archiveContext'] = file_name['archiveContext'] \ + if 'archiveContext' in file_name.keys() else "" + file_name_data['compositePathContext'] = file_name['compositePathContext'] \ + if 'compositePathContext' in file_name.keys() else "" + file_name_data['fileName'] = file_name['fileName'] if 'fileName' in file_name.keys() else "" + comp_unmatched_data['fileNames'].append(file_name_data) + comp_unmatched_data['resourceName'] = comp_unmatched['resourceName'] if 'resourceName' in comp_unmatched.keys() else "" + comp_unmatched_data['matchType'] = comp_unmatched['matchType'] if 'matchType' in comp_unmatched.keys() else "" + comp_unmatched_data['matchTypeLabel'] = comp_unmatched['matchTypeLabel'] if 'matchTypeLabel' in comp_unmatched.keys() else "" + return comp_unmatched_data + +def get_scanned_time(hub_client, codelocations): + """ Retrieve the scanned time from the codelocations data. """ + res = hub_client.session.get(codelocations) + if res.status_code == 200 and res.content: + return json.loads(res.content)['items'][0]['updatedAt'] + # Let's use the 1st element of the list because multiple scans should have taken place at the same time + else: + sys.exit(f"Get codelocations failed for codelocations {codelocations} with status {res.status_code}") + +def get_blackduck_version(hub_client): + url = hub_client.base_url + BLACKDUCK_VERSION_API + res = hub_client.session.get(url) + if res.status_code == 200 and res.content: + return json.loads(res.content)['version'] + else: + sys.exit(f"Get BlackDuck version failed with status {res.status_code}") + +def generate_file_report(hub_client, project_id, version_id, codelocations, copyright_level, format, retries, detect_params=None): + """ + Create a consolidated file report from BlackDuck project version report and notice report. + Remarks: + """ + if not os.path.exists(REPORT_DIR): + os.makedirs(REPORT_DIR) + + # Report headers + report_content['configurationSettings']['scanDateTime'] = get_scanned_time(hub_client, codelocations) + report_content['configurationSettings']['blackDuckVersion'] = get_blackduck_version(hub_client) + report_content['configurationSettings']['detectParameters'] = detect_params + blackduck_link_component_ui_api.replace("{projectId}", project_id).replace("{projectVersionId}", version_id) + report_content['configurationSettings']['linkToBlackDuckProjectVersionInUI'] = \ + hub_client.base_url + blackduck_link_component_ui_api.replace("{projectId}", project_id).replace("{projectVersionId}", version_id) + report_content['configurationSettings']['linkToBlackDuckSnippetMatchInUI'] = \ + hub_client.base_url + blackduck_link_snippet_ui_api.replace("{projectId}", project_id).replace("{projectVersionId}", version_id) \ + + BLACKDUCK_SNIPPET_FILTER + + # Report body - Component BOM, file BOM with Discoveries data + version_report_zip = get_version_detail_report(hub_client, project_id, version_id, retries) + with ZipFile(f"./{version_report_zip}", "r") as vzf: + vzf.extractall() + for i, unzipped_version in enumerate(vzf.namelist()): + if re.search(r"\bversion.+json\b", unzipped_version) is not None: + break + if i + 1 >= len(vzf.namelist()): + sys.exit(f"Version detail file not found in the downloaded report: {version_report_zip}!") + + # Report body - Component BOM report + # Iterated json handling to reduce memory consumption. Remarks: Divided to two file-open sessions for components and files + # with respective file handles, because if otherwise ijson crashes! + with open(f"./{unzipped_version}", "r") as uvf: + for i, comp_bom in enumerate(ijson.items(uvf, 'aggregateBomViewEntries.item')): + comp_data = pull_component_bom(comp_bom) + report_component_bom['bomComponentEntries']['bomComponents'].append(comp_data) + logging.info(f"Number of the reported components {i+1}") + with open(REPORT_DIR + REPORT_COMPONENT_BOM + f".{format}", "w") as cmf: + if format == "json": + cmf.write(json.dumps(report_component_bom)) + else: + cmf.write(json2html.convert(json = json.dumps(report_component_bom))) + report_content['fileInventory']['linkToBomComponentEntries'] = \ + "file://" + os.path.abspath(REPORT_DIR + REPORT_COMPONENT_BOM + f".{format}") + + # Discovery data - licenses, copyrights and unmatched files - is fetched and integrated with file BOM report. + discovery_data = {'discoveries': {'licenses': [], 'copyrights': []}} + discovery_report_zip = get_discovery_report(hub_client, project_id, version_id, retries, copyright_level) + with ZipFile(f"./{discovery_report_zip}", "r") as zlf: + zlf.extractall() + for i, unzipped_discovery in enumerate(zlf.namelist()): + if re.search(r"\bversion-license.+json\b", unzipped_discovery) is not None: + break + if i + 1 >= len(zlf.namelist()): + sys.exit(f"License file not found in downloaded report: {discovery_report_zip}!") + # Do not reuse TextIoWrapper for ijson to handle multiple items. If do, ijson crashes. + with open(f"./{unzipped_discovery}", "r") as ulf: + for i, comp_license in enumerate(ijson.items(ulf, 'componentLicenses.item')): + comp_license_data = pull_discovery_licenses(comp_license) + discovery_data['discoveries']['licenses'].append(comp_license_data) + logging.info(f"Number of the reported discovery licenses {i+1}") + if copyright_level != 0: + with open(f"./{unzipped_discovery}", "r") as ucf: + for i, comp_copyright in enumerate(ijson.items(ucf, 'componentCopyrightTexts.item')): + comp_copyright_data = pull_discovery_copyrights(comp_copyright) + discovery_data['discoveries']['copyrights'].append(comp_copyright_data) + logging.info(f"Number of the reported discovery copyright texts {i+1}") + with open(f"./{unzipped_discovery}", "r") as uuf: + for i, comp_unmatched in enumerate(ijson.items(uuf, 'unmatchedFileData.item')): + comp_unmatched_data = pull_discovery_unmatched(comp_unmatched) + report_file_bom['bomFileEntries']['unmatchedFileDiscoveries'].append(comp_unmatched_data) + logging.info(f"Number of the reported discovery unmatched files texts {i+1}") + + # Report body - Generate file BOM report. Discovery data is integrated. + with open(f"./{unzipped_version}", "r") as uvf: + for i, file_bom in enumerate(ijson.items(uvf, 'detailedFileBomViewEntries.item')): + file_data = pull_file_bom(file_bom) + disc_licenses = list(filter(lambda license_x: + license_x['projectName'] == file_data['projectName'] and + license_x['versionName'] == file_data['projectVersion'], + discovery_data['discoveries']['licenses'])) + if len(disc_licenses) != 0: + disc_license = disc_licenses[0] + for license in disc_license['licenses']: + file_data['licenses'].append(license) + else: + logging.debug(f"No discovery license found for file with this component: {file_data['projectName']} and {file_data['projectVersion']}") + if len(disc_licenses) > 1: + logging.warning(f"More than one discovery license: {disc_licenses} found for {file_data['projectName']}" and file_data['projectVersion']) + + if copyright_level != 0: + origin_name = file_data['channelReleaseExternalNamespace'] + ":" + file_data['channelReleaseExternalId'] + disc_copyrights = list(filter(lambda copyright_x: + copyright_x['originFullName'] == origin_name, + discovery_data['discoveries']['copyrights'])) + if len(disc_copyrights) != 0: + disc_copyright = disc_copyrights[0] + for copyright in disc_copyright['copyrights']['copyrightTexts']: + file_data['copyrights']['copyrightTexts'].append(copyright) + for file_copyright in disc_copyright['copyrights']['fileCopyrightTexts']: + file_data['copyrights']['fileCopyrightTexts'].append(file_copyright) + else: + logging.debug(f"No discovery copyright found for file with this origin: {origin_name}") + if len(disc_copyrights) > 1: + logging.warning(f"More than one discovery copyright: {disc_copyrights} found for the same origin {origin_name}") + + report_file_bom['bomFileEntries']['bomFiles'].append(file_data) + logging.info(f"Number of the reported files {i+1}") + + with open(REPORT_DIR + REPORT_FILE_BOM + f".{format}", "w") as flf: + if format == "json": + flf.write(json.dumps(report_file_bom)) + else: + flf.write(json2html.convert(json = json.dumps(report_file_bom))) + report_content['fileInventory']['linkToBomFileEntries'] = \ + "file://" + os.path.abspath(REPORT_DIR + REPORT_FILE_BOM + f".{format}") + + # Report body - Paths and sizes for folders and files which are not matched by BlackDuck + parent_path = "." + for param in detect_params: + if re.search(rf"{BLACKDUCK_SOURCE_PATH}", param) is not None: + parent_path = param.split("=", 1)[1] + break + logging.info("OS file information for the target source is being traversed and reported.") + matched_paths = [] + for file_bom in report_file_bom['bomFileEntries']['bomFiles']: + # BlackDuck can match internal folders or files within archive files, e.g., .jar or .zip or .tar.gz. + # OS unmatched file function does not search inside of the archived contents as it sounds too much. + if (file_bom['archiveContext']) != "": + continue + else: + matched_paths.append(file_bom['path']) + # Let's remove duplicated path entries. BD may report multiple BOM file entries for the same path. + unique_paths = list(set(matched_paths)) + if len(unique_paths) != len(matched_paths): + logging.warning("BlackDuck component BOM contains duplicated file paths!") + for os_path_data in get_os_path_for_unmatched(parent_path, unique_paths): + report_os_file['unmatchedOsFileEntries']['unmatched'].append(os_path_data) + with open(REPORT_DIR + REPORT_OS_FILE + f".{format}", "w") as osf: + if format == "json": + osf.write(json.dumps(report_os_file)) + else: + osf.write(json2html.convert(json = json.dumps(report_os_file))) + report_content['fileInventory']['linkToUnmatchedOsFileData'] = \ + "file://" + os.path.abspath(REPORT_DIR + REPORT_OS_FILE + f".{format}") + + with open(REPORT_DIR + REPORT_HEADER + f".{format}", "w") as rf: + if format == "json": + rf.write(json.dumps(report_content)) + else: + rf.write(json2html.convert(json = json.dumps(report_content))) + +def main(): + args = parse_parameter() + + try: + if args.copyright_level >= 3: + sys.exit("please provide the copyright level which is either 0 or 1 or 2!") + if (args.report_format).lower() != "json" and (args.report_format).lower() != "html": + sys.exit("Please set either 'json' or 'html' to the report format") + + with open(".restconfig.json", "r") as f: + config = json.load(f) + # Remove last slash if there is, otherwise REST API may fail. + if re.search(r".+/$", config['baseurl']): + bd_url = config['baseurl'][:-1] + else: + bd_url = config['baseurl'] + bd_token = config['api_token'] + bd_insecure = not config['insecure'] + debug = 1 if config['debug'] else 0 + + log_config(debug) + + if not args.skip_detect: + run_detect(args.project, args.version, bd_url, bd_token, args.detect_version, args.detect_parameters) + + hub_client = Client(token=bd_token, + base_url=bd_url, + verify=bd_insecure, + timeout=args.timeout, + retries=args.retries) + + project_id, version_id, codelocations = get_bd_project_data(hub_client, args.project, args.version) + + generate_file_report(hub_client, + project_id, + version_id, + codelocations, + args.copyright_level, + args.report_format.lower(), + args.report_retries, + args.detect_parameters + ) + + except (Exception, BaseException) as err: + logging.error(f"Exception by {str(err)}. See the stack trace") + traceback.print_exc() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/examples/client/copy_kb_component_status_updates.py b/examples/client/copy_kb_component_status_updates.py new file mode 100644 index 00000000..7bf5a763 --- /dev/null +++ b/examples/client/copy_kb_component_status_updates.py @@ -0,0 +1,110 @@ +''' +Created on Jan 29, 2024 +@author: pedapati + +Copyright (C) 2024 Synopsys, Inc. +https://www.synopsys.com + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +This program will Export KB & KB Modified Component Status updates from one Black Duck Server and Import onto another Black Duck Server +''' + +from blackduck import Client + +import requests +import argparse +import json +import logging +import sys +import time +from pprint import pprint + + +NAME = 'copy_kb_component_status_updates.py' +VERSION = '2024-01-29' + +print(f'{NAME} ({VERSION}). Copyright (c) 2023 Synopsys, Inc.') + + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + + +def parse_command_args(): + + parser = argparse.ArgumentParser(sys.argv[0]) + parser.add_argument("-su", "--source-bd-url", required=True, help="Source BD server URL to copy KB component adjustments from e.g. https://your.blackduck.url") + parser.add_argument("-st", "--source-token-file", required=True, help="File containing Source BD access token") + parser.add_argument("-du", "--dest-bd-url", required=True, help="Destination BD server URL to apply KB component adjustments to e.g. https://your.blackduck.url") + parser.add_argument("-dt", "--dest-token-file", required=True, help="File containing Destination BD access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + return parser.parse_args() + + +def main(): + args = parse_command_args() + ## Step 1 Source Black Duck Server - Authentication + with open(args.source_token_file, 'r') as tf: + access_token = tf.readline().strip() + bd1 = Client(base_url=args.source_bd_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + ## Step 2 Destination Black Duck Server - Authentication + with open(args.dest_token_file, 'r') as tf: + access_token1 = tf.readline().strip() + bd2 = Client(base_url=args.dest_bd_url, token=access_token1, verify=args.no_verify, timeout=60.0, retries=4) + ## Step 3 Source Black Duck Server - Get KB Components with Status Updates + headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} + get_comp_url = f"{bd1.base_url}/api/components?filter=componentApprovalStatus%3Ain_review&filter=componentApprovalStatus%3Areviewed&filter=componentApprovalStatus%3Aapproved&filter=componentApprovalStatus%3Alimited_approval&filter=componentApprovalStatus%3Arejected&filter=componentApprovalStatus%3Adeprecated&filter=componentSource%3Akb_and_kb_modified&limit=25&offset=0" + get_comp_json = bd1.session.get(get_comp_url, headers=headers).json() + total = str(get_comp_json["totalCount"]) + print("Found " + total + " KB components with status updates") + print() + for component in get_comp_json["items"]: + comp_name = component['name'] + # comp_url = component['url'] + comp_status = component['approvalStatus'] + comp_url = component['_meta']['href'] + comp_id = comp_url.split("/")[-1] + print("Updating KB Component " + comp_name + " status to " + comp_status) + ## Step 4 Destination Black Duck Server - Update KB Components with Status Updates + headers = {'Content-Type': 'application/vnd.blackducksoftware.component-detail-4+json', 'Accept': 'application/vnd.blackducksoftware.component-detail-4+json'} + put_data = {"name": comp_name, "approvalStatus": comp_status} + put_comp_url = f"{bd2.base_url}/api/components/{comp_id}" + update_comp_results=bd2.session.put(put_comp_url, headers=headers, data=json.dumps(put_data)) + if update_comp_results.status_code == 200: + message = f"{update_comp_results}" + print("Successfully updated status of " + comp_name + " to " + comp_status) + print() + else: + message = f"{update_comp_results.json()}" + print("Updating status FAILED with error message:") + print() + logging.debug({message}) + print() + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/examples/client/crypto-to-custom.py b/examples/client/crypto-to-custom.py new file mode 100644 index 00000000..1ecf6ecd --- /dev/null +++ b/examples/client/crypto-to-custom.py @@ -0,0 +1,159 @@ +''' +Created on October 12, 2023 +@author: kumykov + +Copyright (C) 2023 Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +This script is provided as an example of populating custom field data +based on BOM components crypto information. +The goal is to enable policy functionality that would be triggered by +cryptographic features of a component. + +The script will analyze ciphers included in a component and will set +a BOM Component custom field value to reflect that a known weakness is present. + +Requirements + +- python3 version 3.8 or newer recommended +- the following packages are used by the script and should be installed + prior to use: + argparse + blackduck + logging + sys + json + pprint +- Blackduck instance +- API token with sufficient privileges to perform project version phase + change. + +Install python packages with the following command: + + pip3 install argparse blackduck logging sys json pprint + +Using + +Script expects a boolean custom field labeled "BadCrypto" on a BOM Component. +A policy that is triggered by BadCrypto custom field value used to visualize +results. + +usage: crypto-to-custom.py [-h] -u BASE_URL -t TOKEN_FILE -pn PROJECT_NAME -vn VERSION_NAME [-nv] [--reset] + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -pn PROJECT_NAME, --project-name PROJECT_NAME + Project Name + -vn VERSION_NAME, --version-name VERSION_NAME + Version Name + -nv, --no-verify Disable TLS certificate verification + --reset Undo the changes made by this script + + +''' + +import argparse +from blackduck import Client +from pprint import pprint +import logging +import sys +import json + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.INFO) +logging.getLogger("urllib3").setLevel(logging.INFO) +logging.getLogger("blackduck").setLevel(logging.INFO) + +def find_project_by_name(project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + +def parse_command_args(): + + parser = argparse.ArgumentParser("crypto-to-custom.py") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-pn", "--project-name", required=True, help="Project Name") + parser.add_argument("-vn", "--version-name", required=True, help="Version Name") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("--reset", action='store_true', help="Undo the changes made by this script") + return parser.parse_args() + +def set_custom_field(field, url, value): + payload = {"fields": [{"customField": field['_meta']['href'],"values": value}]} + headers = {"Accept": "application/vnd.blackducksoftware.bill-of-materials-6+json", + "Content-Type": "application/vnd.blackducksoftware.bill-of-materials-6+json"} + response = bd.session.put(url, data=json.dumps(payload), headers=headers) + print(response) + +def process_project_version(args): + project = find_project_by_name(args.project_name) + version = find_project_version_by_name(project, args.version_name) + + components = bd.get_resource('components',version) + for component in components: + print (component['componentName'], component['componentVersionName']) + custom_fields = bd.get_resource('custom-fields',component, items=False) + custom_fields_url = custom_fields['_meta']['href'] + c = [x for x in custom_fields['items'] if x['label'] == 'BadCrypto'][0] + resources = bd.list_resources(component) + if 'crypto-algorithms' in resources.keys(): + crypto_algorithms = bd.get_resource('crypto-algorithms', component) + for crypto in crypto_algorithms: + if crypto['knownWeaknesses']: + pprint('Has Weakness') + value = ['true'] + if args.reset: + value = [] + set_custom_field(c, custom_fields_url, value=value) + break + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + global bd + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + + process_project_version(args) + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/examples/client/file_hierarchy_report.py b/examples/client/file_hierarchy_report.py new file mode 100644 index 00000000..d80f1d45 --- /dev/null +++ b/examples/client/file_hierarchy_report.py @@ -0,0 +1,309 @@ +''' +Created on June 25, 2024 + +@author: dnichol and kumykov + +Generate version detail reports (source and components) and consolidate information on source matches, with license +and component matched. Removes matches found underneith other matched components in the source tree (configurable). + +Copyright (C) 2023 Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +''' + +import argparse +import csv +import logging +import sys +import io +import time +import json +import traceback +from blackduck import Client +from zipfile import ZipFile +from pprint import pprint + +program_description = \ +'''Generate version detail reports (source and components) and consolidate information on source matches, with license +and component matched. Removes matches found underneath other matched components in the source tree (configurable). + +This script assumes a project version exists and has scans associated with it (i.e. the project is not scanned as part of this process). + +''' + +# BD report general +BLACKDUCK_VERSION_MEDIATYPE = "application/vnd.blackducksoftware.status-4+json" +BLACKDUCK_VERSION_API = "/api/current-version" +# Retries to wait for BD report creation. RETRY_LIMIT can be overwritten by the script parameter. +RETRY_LIMIT = 30 +RETRY_TIMER = 30 + +def log_config(debug): + if debug: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.DEBUG) + else: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.INFO) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("blackduck").setLevel(logging.WARNING) + +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + assert len(projects) == 1, f"Project {project_name} not found." + return projects[0] + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + assert len(versions) == 1, f"Project version {version_name} for project {project['name']} not found" + return versions[0] + +def create_version_details_report(bd, version): + version_reports_url = bd.list_resources(version).get('versionReport') + post_data = { + 'reportFormat' : 'JSON', + 'locale' : 'en_US', + 'versionId': version['_meta']['href'].split("/")[-1], + 'categories' : [ 'COMPONENTS', 'FILES' ] # Generating "project version" report including components and files + } + + bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json" + r = bd.session.post(version_reports_url, json=post_data) + if (r.status_code == 403): + logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") + r.raise_for_status() + location = r.headers.get('Location') + assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" + return location + +def download_report(bd, location, retries, timeout): + report_id = location.split("/")[-1] + logging.debug(f"Report location {location}") + url_data = location.split('/') + url_data.pop(4) + url_data.pop(4) + download_link = '/'.join(url_data) + logging.debug(f"Report Download link {download_link}") + if retries: + logging.debug(f"Retrieving generated report for {location} via {download_link}") + response = bd.session.get(location) + report_status = response.json().get('status', 'Not Ready') + if response.status_code == 200 and report_status == 'COMPLETED': + response = bd.session.get(download_link, headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + if response.status_code == 200: + return response.content + else: + logging.error("Ruh-roh, not sure what happened here") + return None + else: + logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {timeout} seconds then retrying...") + time.sleep(timeout) + retries -= 1 + return download_report(bd, location, retries, timeout) + else: + logging.debug(f"Failed to retrieve report {report_id} after multiple retries") + return None + +def get_blackduck_version(hub_client): + url = hub_client.base_url + BLACKDUCK_VERSION_API + res = hub_client.session.get(url) + if res.status_code == 200 and res.content: + return json.loads(res.content)['version'] + else: + sys.exit(f"Get BlackDuck version failed with status {res.status_code}") + +def reduce(path_set): + path_set.sort() + for path in path_set: + if len(path) < 3: + continue + index = path_set.index(path) + while index + 1 < len(path_set) and path in path_set[index+1]: + logging.debug(f"{path} is in {path_set[index+1]} deleting the sub-path from the list") + path_set.pop(index+1) + return path_set + +def trim_version_report(version_report, reduced_path_set): + file_bom_entries = version_report['detailedFileBomViewEntries'] + aggregate_bom_view_entries = version_report['aggregateBomViewEntries'] + + reduced_file_bom_entries = [e for e in file_bom_entries if f"{e.get('archiveContext', '')}!{e['path']}" in reduced_path_set] + version_report['detailedFileBomViewEntries'] = reduced_file_bom_entries + + component_identifiers = [f"{e['projectId']}:{e['versionId']}" for e in reduced_file_bom_entries] + deduplicated = list(dict.fromkeys(component_identifiers)) + + reduced_aggregate_bom_view_entries = [e for e in aggregate_bom_view_entries if f"{e['producerProject']['id']}:{e['producerReleases'][0]['id']}" in deduplicated] + version_report['aggregateBomViewEntries'] = reduced_aggregate_bom_view_entries + +''' + +CSV output details + +component name = aggregateBomViewEntries[].producerProject.name +version name = aggregateBomViewEntries[].producerReleases[0].version +license = licenses[].licenseDisplay +file path = extract from detailedFileBomViewEntries +match type = aggregateBomViewEntries[].matchTypes +review status = aggregateBomViewEntries[].reviewSummary.reviewStatus + +''' +def get_csv_fieldnames(): + return ['component name', 'version name', 'license', 'match type', 'review status'] + +def get_csv_data(version_report, keep_dupes): + csv_data = list() + components = list() + for bom_view_entry in version_report['aggregateBomViewEntries']: + entry = dict() + entry['component name'] = bom_view_entry['producerProject']['name'] + entry['version name'] = bom_view_entry['producerReleases'][0]['version'] + entry['license'] = bom_view_entry['licenses'][0]['licenseDisplay'].replace(' AND ',';').replace('(','').replace(')','') + pid = bom_view_entry['producerProject']['id'] + vid = bom_view_entry['producerReleases'][0]['id'] + #path_list = [p['path'] for p in version_report['detailedFileBomViewEntries'] if p['projectId'] == pid and p['versionId'] == vid] + #entry['file path'] = ';'.join(path_list) + entry['match type'] = ';'.join(bom_view_entry['matchTypes']) + entry['review status'] = bom_view_entry['reviewSummary']['reviewStatus'] + + # Only add if this component was not previously added. + composite_key = pid + vid + if composite_key not in components: + csv_data.append(entry) + components.append(composite_key) + if keep_dupes: + return csv_data + else: + return remove_duplicates(csv_data) + +def remove_duplicates(data): + # Put data into buckets by version + buckets = dict() + for row in data: + name = row['component name'].lower() + version = row['version name'] + if not version in buckets: + buckets[version] = [row] + else: + buckets[version].append(row) + # Run reduction process for component names that start with existing component name + # This process will ignore case in component names + for set in buckets.values(): + set.sort(key = lambda d: d['component name'].lower()) + for row in set: + index = set.index(row) + name = row['component name'].lower() + while index + 1 < len(set) and set[index+1]['component name'].lower().startswith(name): + set.pop(index+1) + reduced_data = list() + for b in buckets.values(): + reduced_data.extend(b) + return reduced_data + +def write_output_file(version_report, output_file, keep_dupes): + if output_file.lower().endswith(".csv"): + logging.info(f"Writing CSV output into {output_file}") + field_names = get_csv_fieldnames() + with open(output_file, "w") as f: + writer = csv.DictWriter(f, fieldnames = field_names, extrasaction = 'ignore',quoting=csv.QUOTE_ALL) # TODO + writer.writeheader() + writer.writerows(get_csv_data(version_report, keep_dupes)) + return + # If it's neither, then .json + if not output_file.lower().endswith(".json"): + output_file += ".json" + logging.info(f"Writing JSON output into {output_file}") + with open(output_file,"w") as f: + json.dump(version_report, f) + +def parse_command_args(): + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") + parser.add_argument("-pn", "--project-name", required=True, help="Project Name") + parser.add_argument("-pv", "--project-version-name", required=True, help="Project Version Name") + parser.add_argument("-o", "--output-file", required=False, help="File name to write output. File extension determines format .json and .csv, json is the default.") + parser.add_argument("-kd", "--keep-dupes", action='store_true', help="Do not reduce CVS data by fuzzy matching component names") + parser.add_argument("-kh", "--keep_hierarchy", action='store_true', help="Set to keep all entries in the sources report. Will not remove components found under others.") + parser.add_argument("--report-retries", metavar="", type=int, default=RETRY_LIMIT, help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.") + parser.add_argument("--report-timeout", metavar="", type=int, default=RETRY_TIMER, help="Wait time between subsequent download attempts.") + parser.add_argument("--timeout", metavar="", type=int, default=60, help="Timeout for REST-API. Some API may take longer than the default 60 seconds") + parser.add_argument("--retries", metavar="", type=int, default=4, help="Retries for REST-API. Some API may need more retries than the default 4 times") + return parser.parse_args() + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + token = tf.readline().strip() + output_file = args.output_file + if not args.output_file: + output_file = f"{args.project_name}-{args.project_version_name}.json".replace(" ","_") + try: + log_config(args.debug) + hub_client = Client(token=token, + base_url=args.base_url, + verify=args.no_verify, + timeout=args.timeout, + retries=args.retries) + + project = find_project_by_name(hub_client, args.project_name) + version = find_project_version_by_name(hub_client, project, args.project_version_name) + location = create_version_details_report(hub_client, version) + report_zip = download_report(hub_client, location, args.report_retries, args.report_timeout) + logging.debug(f"Deleting report from Black Duck {hub_client.session.delete(location)}") + zip=ZipFile(io.BytesIO(report_zip), "r") + pprint(zip.namelist()) + report_data = {name: zip.read(name) for name in zip.namelist()} + filename = [i for i in report_data.keys() if i.endswith(".json")][0] + version_report = json.loads(report_data[filename]) + with open("out.json", "w") as f: + json.dump(version_report, f) + # TODO items + # Process file section of report data to identify primary paths + path_set = [f"{entry.get('archiveContext', '')}!{entry['path']}" for entry in version_report['detailedFileBomViewEntries']] + reduced_path_set = reduce(path_set.copy()) + logging.info(f"{len(path_set)-len(reduced_path_set)} path entries were scrubbed from the dataset.") + + # Remove component entries that correspond to removed path entries. + + logging.info(f"Original dataset contains {len(version_report['aggregateBomViewEntries'])} bom entries and {len(version_report['detailedFileBomViewEntries'])} file view entries") + if not args.keep_hierarchy: + trim_version_report(version_report, reduced_path_set) + logging.info(f"Truncated dataset contains {len(version_report['aggregateBomViewEntries'])} bom entries and {len(version_report['detailedFileBomViewEntries'])} file view entries") + + write_output_file(version_report, output_file, args.keep_dupes) + + # Combine component data with selected file data + # Output result with CSV anf JSON as options. + + + + except (Exception, BaseException) as err: + logging.error(f"Exception by {str(err)}. See the stack trace") + traceback.print_exc() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/examples/client/generate_sbom.py b/examples/client/generate_sbom.py index 70842419..d256d152 100644 --- a/examples/client/generate_sbom.py +++ b/examples/client/generate_sbom.py @@ -45,10 +45,10 @@ class FailedReportDownload(Exception): parser.add_argument("token_file", help="containing access token") parser.add_argument("project_name") parser.add_argument("version_name") -parser.add_argument("-z", "--zip_file_name", default="reports.zip") -parser.add_argument("-t", "--type", type=str, nargs='?', default="SPDX_22", choices=["SPDX_22", "CYCLONEDX_13"], help="Choose the type of SBOM report") +parser.add_argument("-t", "--type", type=str, nargs='?', default="SPDX_23", choices=["SPDX_22", "SPDX_23","SPDX_30", "CYCLONEDX_13", "CYCLONEDX_14","CYCLONEDX_16", "CYCLONEDX_15"], help="Choose the type of SBOM report") parser.add_argument('-r', '--retries', default=4, type=int, help="How many times to retry downloading the report, i.e. wait for the report to be generated") -parser.add_argument('-s', '--sleep_time', default=5, type=int, help="The amount of time to sleep in-between (re-)tries to download the report") +parser.add_argument('-s', '--sleep_seconds', default=60, type=int, help="The amount of time to sleep in-between (re-)tries to download the report") +parser.add_argument('--include-subprojects', dest='include_subprojects', action='store_false', help="whether subprojects should be included") parser.add_argument('--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") args = parser.parse_args() @@ -75,8 +75,8 @@ def download_report(bd_client, location, filename, retries=args.retries): logging.error("Ruh-roh, not sure what happened here") else: logging.debug(f"Failed to retrieve report {report_id}, report status: {report_status}") - logging.debug("Probably not ready yet, waiting 5 seconds then retrying...") - time.sleep(args.sleep_time) + logging.debug(f"Probably not ready yet, waiting {args.sleep_seconds} seconds then retrying...") + time.sleep(args.sleep_seconds) retries -= 1 download_report(bd_client, location, filename, retries) else: @@ -105,16 +105,20 @@ def download_report(bd_client, location, filename, retries=args.retries): post_data = { 'reportFormat': "JSON", - 'reportType': 'SBOM', - 'sbomType': args.type, + 'sbomType': args.type, + 'includeSubprojects': args.include_subprojects } sbom_reports_url = version['_meta']['href'] + "/sbom-reports" +bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json" r = bd.session.post(sbom_reports_url, json=post_data) +if (r.status_code == 403): + logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") r.raise_for_status() location = r.headers.get('Location') +print(f"location {location}") assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" logging.debug(f"Created SBOM report of type {args.type} for project {args.project_name}, version {args.version_name} at location {location}") -download_report(bd, location, args.zip_file_name) +download_report(bd, location, f"{args.project_name}({args.version_name}).zip") diff --git a/examples/client/generate_vuln_status_report.py b/examples/client/generate_vuln_status_report.py new file mode 100644 index 00000000..cecba564 --- /dev/null +++ b/examples/client/generate_vuln_status_report.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python + +''' +Copyright (C) 2021 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +''' +import argparse +from datetime import datetime +import json +import logging +import requests +import sys +import time +import csv + +from blackduck import Client + +DEFAULT_OUTPUT_FILE="vuln_status_report.csv" + +parser = argparse.ArgumentParser("Generate a vulnerability status report") +parser.add_argument("--base-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", help="containing access token") +parser.add_argument("--projects", nargs="+", help="The list of projects to include in the report") +parser.add_argument('-t', '--tries', default=10, type=int, help="How many times to retry downloading the report, i.e. wait for the report to be generated") +parser.add_argument("-o", "--output-file-name", + dest="file_name", + default=DEFAULT_OUTPUT_FILE, + help=f"Name of the output file (default: {DEFAULT_OUTPUT_FILE})") +parser.add_argument("--no-verify", + dest='verify', + action='store_false', + help="disable TLS certificate verification") +args = parser.parse_args() + + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stdout, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +class FailedReportDownload(Exception): + pass + +def download_vuln_report(bd_client, location, filename, report_format, retries=args.tries): + if retries: + report_status = bd_client.session.get(location).json() + if report_status['status'] == 'COMPLETED': + download_url = bd_client.list_resources(report_status)['download'] + contents_url = download_url + "/contents" + report_contents = bd_client.session.get(contents_url).json() + if report_format == 'JSON': + with open(filename, 'w') as f: + json.dump(report_contents, f, indent=3) + logging.info(f"Wrote vulnerability status report contents to {filename}") + elif report_format == 'CSV': + csv_data = report_contents['reportContent'][0]['fileContent'] + with open(filename, 'w') as f: + f.write(csv_data) + logging.info(f"Wrote vulnerability status report contents to {filename}") + else: + logging.error(f"Unrecognized format ({report_format}) given. Exiting") + + else: + sleep_time = 25 + retries -= 1 + logging.debug(f"Report is not ready to download yet, waiting {sleep_time} seconds and then retrying {retries} more times") + time.sleep(sleep_time) + download_vuln_report(bd_client, location, filename, report_format, retries) + else: + raise FailedReportDownload(f"Failed to retrieve report from {location} after {retries} attempts") + + +def get_projects(client, project_names): + print (project_names) + '''Given a list of project names return a list of the corresponding project URLs''' + project_urls = list() + for project in client.get_items("/api/projects"): + if project['name'] in project_names: + project_urls.append(project['_meta']['href']) + return project_urls + +def augment_filename(filename): + if filename.endswith('.csv'): + index = filename.index('.csv') + return filename[:index] + '_augmented' + filename[index:] + else: + return filename + '_augmented.csv' + +def correct_vuln_ids(bd, filename, new_filename): + logging.debug(f"Generating file with augmented vuln ids as {new_filename}") + input = open(filename, 'r') + reader = csv.DictReader(input) + fieldnames = reader.fieldnames + rowcount = 0 + with open(new_filename, 'w') as output: + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + for row in reader: + vuln_id = row['Vulnerability id'] + related_vuln_id = get_related_vuln_id(bd, vuln_id) + if related_vuln_id: + correct_vuln_id = f"{vuln_id} ({related_vuln_id})" + else: + correct_vuln_id = vuln_id + row['Vulnerability id'] = correct_vuln_id + writer.writerow(row) + rowcount+=1 + if rowcount % 100 == 0: + logging.debug(f"{rowcount:15} rows written into {new_filename}") + logging.debug(f"Total of {rowcount} rows written into {new_filename}") + logging.info(f"Wrote vulnerability status report contents to {filename}") + +def get_related_vuln_id(bd, vuln_id): + related_id = None + if vuln_id.startswith('BDSA') and 'CVE' not in vuln_id: + vuln_data = bd.get_json(f"/api/vulnerabilities/{vuln_id}") + vuln_resources = bd.list_resources(vuln_data) + related_url = vuln_resources.get('related-vulnerability', None) + if related_url: + related_id = related_url.split('/')[-1:][0] + return related_id + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client( + base_url=args.base_url, + token=access_token, + verify=args.verify +) + +project_urls = get_projects(bd, args.projects) +logging.debug(f"Generating vulnerability status report for the following projects: {args.projects}") +logging.debug(f"Project list resulted in following project URLs {project_urls}") +post_data = { + 'reportFormat': 'CSV', + 'projects': project_urls, + 'locale': 'en_US' +} + +try: + r = bd.session.post("/api/vulnerability-status-reports", json=post_data) + r.raise_for_status() + report_url = r.headers['Location'] + logging.debug(f"created vulnerability status report {report_url}") +except requests.HTTPError as err: + # more fine grained error handling here; otherwise: + bd.http_error_handler(err) + logging.error("Failed to generate the report") + sys.exit(1) + +download_vuln_report(bd, report_url, args.file_name, 'CSV', retries=args.tries) + +correct_vuln_ids(bd, args.file_name, augment_filename(args.file_name)) + + + + + + + + + diff --git a/examples/client/get_bom_component_vuln_info.py b/examples/client/get_bom_component_vuln_info.py index 54e4b844..1da4fb3c 100644 --- a/examples/client/get_bom_component_vuln_info.py +++ b/examples/client/get_bom_component_vuln_info.py @@ -48,21 +48,39 @@ all_bom_component_vulns = [] -for bom_component_vuln in bd.get_resource('vulnerable-components', version): - vuln_name = bom_component_vuln['vulnerabilityWithRemediation']['vulnerabilityName'] - vuln_source = bom_component_vuln['vulnerabilityWithRemediation']['source'] +# version of API to call +api_version = 8 + +media_type = "application/vnd.blackducksoftware.bill-of-materials-" + str(api_version) + "+json" +#media_type = "application/json" + +# lower case keys +lc_keys = {} +lc_keys['accept'] = media_type +lc_keys['content-type'] = media_type + +# keyword arguments to pass +kwargs={} +kwargs['headers'] = lc_keys + +for bom_component_vuln in bd.get_resource('vulnerable-components', version, **kwargs): + vulnerabilities = bd.get_resource('vulnerabilities', bom_component_vuln) upgrade_guidance = bd.get_json(f"{bom_component_vuln['componentVersion']}/upgrade-guidance") bom_component_vuln['upgrade_guidance'] = upgrade_guidance + all_bom_component_vulns.append(bom_component_vuln) + #for vuln in vulnerabilities: + #pprint(vuln) + #vuln_name = vuln['name'] + #vuln_source = vuln['source'] - vuln_details = bd.get_json(f"/api/vulnerabilities/{vuln_name}") - bom_component_vuln['vulnerability_details'] = vuln_details + #vuln_details = bd.get_json(f"/api/vulnerabilities/{vuln_name}") + #bom_component_vuln['vulnerability_details'] = vuln_details - if 'related-vulnerability' in bd.list_resources(vuln_details): - related_vuln = bd.get_resource("related-vulnerability", vuln_details, items=False) - else: - related_vuln = None - bom_component_vuln['related_vulnerability'] = related_vuln - all_bom_component_vulns.append(bom_component_vuln) + #if 'related-vulnerability' in bd.list_resources(vuln_details): + # related_vuln = bd.get_resource("related-vulnerability", vuln_details, items=False) + #else: + # related_vuln = None + #bom_component_vuln['related_vulnerability'] = related_vuln if args.csv_file: '''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints @@ -73,13 +91,13 @@ with open(args.csv_file, 'w') as csv_f: field_names = [ 'Vulnerability Name', - 'Vulnerability Description', + #'Vulnerability Description', 'Remediation Status', 'Component', 'Component Version', - 'Exploit Available', - 'Workaround Available', - 'Solution Available', + #'Exploit Available', + #'Workaround Available', + #'Solution Available', 'Upgrade Guidance - short term', 'Upgrade Guidance - long term', ] @@ -87,14 +105,14 @@ writer.writeheader() for comp_vuln in all_bom_component_vulns: row_data = { - 'Vulnerability Name': comp_vuln['vulnerabilityWithRemediation']['vulnerabilityName'], - 'Vulnerability Description': comp_vuln['vulnerabilityWithRemediation']['description'], - 'Remediation Status': comp_vuln['vulnerabilityWithRemediation']['remediationStatus'], + 'Vulnerability Name': comp_vuln['vulnerability']['vulnerabilityId'], + #'Vulnerability Description': comp_vuln['vulnerabilityWithRemediation']['description'], + 'Remediation Status': comp_vuln['vulnerability']['remediationStatus'], 'Component': comp_vuln['componentName'], 'Component Version': comp_vuln['componentVersionName'], - 'Exploit Available': comp_vuln['vulnerability_details'].get('exploitPublishDate', 'None available'), - 'Workaround Available': comp_vuln['vulnerability_details'].get('workaround', 'None available'), - 'Solution Available': comp_vuln['vulnerability_details'].get('solution', 'None available'), + #'Exploit Available': comp_vuln['vulnerability_details'].get('exploitPublishDate', 'None available'), + #'Workaround Available': comp_vuln['vulnerability_details'].get('workaround', 'None available'), + #'Solution Available': comp_vuln['vulnerability_details'].get('solution', 'None available'), 'Upgrade Guidance - short term': comp_vuln['upgrade_guidance'].get('shortTerm', 'None available'), 'Upgrade Guidance - long term': comp_vuln['upgrade_guidance'].get('longTerm', 'None available') } diff --git a/examples/client/get_project_data.py b/examples/client/get_project_data.py new file mode 100644 index 00000000..89bf61c8 --- /dev/null +++ b/examples/client/get_project_data.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +''' +Created: Apr 2, 2024 +Author: @kumykov + +Copyright (c) 2024, Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +usage: get_project_data.py [-h] -u BASE_URL -t TOKEN_FILE [-nv] -p SOURCE_PROJECT -v SOURCE_VERSION + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -nv, --no-verify Disable TLS certificate verification + -p SOURCE_PROJECT, --project SOURCE_PROJECT + Project Name + -v SOURCE_VERSION, --version SOURCE_VERSION + Project Version Name + +Black Duck examples collection + + +''' +import argparse +import io +import json +import sys +import logging +import time + +from blackduck import Client +from pprint import pprint + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + + +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'].casefold() == project_name.casefold()] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + +def get_project_data(bd, args): + project = find_project_by_name(bd, args.project) + version = find_project_version_by_name(bd, project, args.version) + if not version: + logging.error(f"Source project {args.project} : {args.version} not found. Exiting.") + sys.exit(1) + logging.info(f"Located source project {args.project} : {args.version}") + return bd.get_resource('components', version) + + +def parse_command_args(): + parser = argparse.ArgumentParser(prog = "get_project_data.py", description="Generate and download SBOM and upload to the target project version", epilog="Blackduck examples collection") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-p", "--project", required=True, help="Project Name") + parser.add_argument("-v", "--version", required=True, help="Project Version Name") + + return parser.parse_args() + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + components = get_project_data(bd, args) + for component in components: + # pprint (component) + print (f"{component['componentName']} {component['componentVersionName']} {component['licenses'][0]['licenseDisplay']}") + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/client/get_project_vulnerabilites_as_csv.py b/examples/client/get_project_vulnerabilites_as_csv.py index e3be492d..8dfe89a8 100644 --- a/examples/client/get_project_vulnerabilites_as_csv.py +++ b/examples/client/get_project_vulnerabilites_as_csv.py @@ -27,7 +27,7 @@ ) def strip_newline(str): - return str.replace('\r', '').replace('\n', ' ') + return str.replace('\r', '').replace('\n', '\\n') def match_component(selected_components, component): if (len(selected_components) == 0): diff --git a/examples/client/get_scan_missed_import_event.py b/examples/client/get_scan_missed_import_event.py new file mode 100644 index 00000000..6266e240 --- /dev/null +++ b/examples/client/get_scan_missed_import_event.py @@ -0,0 +1,96 @@ +""" +Created on july 11, 2024 +@author: Dinesh Ravi +Gather list of non matched components where blackduck could not able have match event with their kb for the bdio codelocation type +""" +from blackduck import Client + +import argparse + +import logging +import json +# py get_scan_missed_import_event.py --base-url=https://blackduck.company.com --token-file=.pt --project=ASTERIX2CLU3D_PR +# OGRAM --version=AED2_ANDROID_S_2_2024-09-21_00-32 --company=company --no-verify > missing.txt +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser("Get the BOM components for a given project-version and the license details for each BOM component") +parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token") +parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") +parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") +parser.add_argument("--company", dest='company_name', required=True, help="modules that contains the company name for separation") +parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification") +args = parser.parse_args() + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.base_url, token=access_token, verify=args.verify) + +params = { + 'q': [f"name:{args.project_name}"] +} + +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +params = { + 'q': [f"versionName:{args.version_name}"] +} +versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name] +assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}" +version = versions[0] + +print(f"Found {project['name']}:{version['versionName']}") +hashset_company = set() +hashset_other = set() +# =================== +# print(version) +# params = { +# 'q': [f"name:*bdio*"] +# } +codelocations=[codelocation for codelocation in bd.get_resource('codelocations',version) if 'bdio' in codelocation['name']] +# print(codelocations[0]) +# codelocation=codelocations[0] +for codelocation in codelocations: + scans=bd.get_resource('scans',codelocation) + for scan in scans: + events=[events for events in bd.get_resource('component-import-events',scan) if events['event']=="COMPONENT_MAPPING_FAILED"] + if len(events)>0: + print("============================") + print(f"codelocation_name: {codelocation['name']}") + print(f"matchCount: {scan['matchCount']}") + print(f"missing: {len(events)}") + + for i,event in enumerate(events,start=1): + print(f"--------------{i}") + externalId=event['externalId'] + print(f"externalId: {externalId}") + print(f"importComponentName: {event['importComponentName']}") + print(f"importComponentVersionName: {event['importComponentVersionName']}") + if args.company_name in externalId: + hashset_company.add(externalId) + else: + hashset_other.add(externalId) + + # {'event': 'COMPONENT_MAPPING_FAILED', + # 'importComponentName': 'rsi-common-lib', + # 'importComponentVersionName': '0.2.14', + # 'externalId': 'com.company.aed2:rsi-common-lib:0.2.14', + # 'failureReason': 'Unable to map scanned component version to Black Duck project version because no mapping is present for the given external identifier'} +print("============================") +sorted_company=sorted(hashset_company) +sorted_other=sorted(hashset_other) + +print(f"========total missing Other components to get foss report: {len(hashset_other)}========") +for i,missing in enumerate(sorted_other,start=1): + print(f"{i} {missing}") +print(f"========total missing {args.company_name} components to manually add: {len(hashset_company)}========") +for i,missing in enumerate(sorted_company,start=1): + print(f"{i} {missing}") +total_missing=len(sorted_company)+len(sorted_other) +print(f"========EB:{len(sorted_company)}+Other:{len(sorted_other)}={total_missing}========") diff --git a/examples/client/list_projects.py b/examples/client/list_projects.py new file mode 100644 index 00000000..91897f4d --- /dev/null +++ b/examples/client/list_projects.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +''' +Copyright (C) 2023 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +List projects and project owners + +usage: list_projects.py [-h] -u BASE_URL -t TOKEN_FILE [-nv] + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + containing access token + -nv, --no-verify disable TLS certificate verification + +''' +import argparse +import json +import logging +import sys +import arrow +import csv + +from blackduck import Client +from pprint import pprint + +parser = argparse.ArgumentParser("List projects and project owners") +parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", dest='token_file', required=True, help="containing access token") +parser.add_argument("-nv", "--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification") +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client( + base_url=args.base_url, + token=access_token, + verify=args.verify +) + +headers = {} +projects = bd.get_resource('projects') + +def get_user_name(url): + if url: + data = bd.get_json(url) + return (data['userName']) + else: + return None + +fieldnames = ['Project Name', + 'Project Owner', + 'Project Created By', + 'Project Updated By'] + +file_name = 'projects_by_owners.csv' + +with open(file_name, 'w') as output: + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + + for project in projects: + project_name = project['name'] + project_owner = project.get('projectOwner', None) + project_owner_name = get_user_name(project_owner) + project_created_by = project.get('createdByUser', None) + project_created_by_name = get_user_name(project_created_by) + project_updated_by = project.get('updatedByUser', None) + project_updated_by_name = get_user_name(project_updated_by) + project_users = bd.get_resource('users', project) + row = dict() + row['Project Name'] = project_name + row['Project Owner'] = project_owner_name + row['Project Created By'] = project_created_by_name + row['Project Updated By'] = project_updated_by_name + writer.writerow(row) + + logging.info(f"Output file {file_name} written") \ No newline at end of file diff --git a/examples/client/match_snippet.py b/examples/client/match_snippet.py new file mode 100644 index 00000000..969bce89 --- /dev/null +++ b/examples/client/match_snippet.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +''' +Copyright (C) 2024 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +usage: match_snippets.py [-h] --base-url BASE_URL --token-file TOKEN_FILE [--no-verify] [--input INPUT] + +options: + -h, --help show this help message and exit + --base-url BASE_URL Hub server URL e.g. https://your.blackduck.url + --token-file TOKEN_FILE + containing access token + --no-verify disable TLS certificate verification + --input INPUT File containing code snippet or stdin + +Match a snippet of a code. +This functionality requires 'Generative AI Compliance' option licenses + + +Examples: + +Curl file content from github and match it against Black Duck KB +and format the output using jq utility + curl https://raw.githubusercontent.com/apache/kafka/trunk/shell/src/main/java/org/apache/kafka/shell/state/MetadataShellState.java | \ + python3 examples/client/match_snippet.py --base-url=$BD_URL --token-file=<(echo $API_TOKEN) --no-verify | \ + jq . + +This will produce something like: +{ + "snippetMatches": { + "PERMISSIVE": [ + { + "projectName": "Apache Kafka", + "releaseVersion": "3.5.0", + "licenseDefinition": { + "name": "Apache License 2.0", + "spdxId": "Apache-2.0", + "ownership": "OPEN_SOURCE", + "licenseDisplayName": "Apache License 2.0" +. . . + +''' +import argparse +import json +import logging +import sys + +from blackduck import Client + +parser = argparse.ArgumentParser('match_snippets.py') +parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token") +parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("--input", required=False, help="File containing code snippet or stdin") +args = parser.parse_args() + + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client( + base_url=args.base_url, + token=access_token, + verify=args.verify +) + +if args.input: + with open(args.input, 'r') as content_file: + content = content_file.read() +else: + with sys.stdin as content_file: + content = content_file.read() + +endpoint='/api/snippet-matching' +headers = {"Content-Type": "text/plain"} + +response = bd.session.post(url=endpoint, headers=headers, data=content) +if response.ok: + data = response.json() + import json + print(json.dumps(data)) \ No newline at end of file diff --git a/examples/client/multi-image/README.md b/examples/client/multi-image/README.md new file mode 100644 index 00000000..50db637b --- /dev/null +++ b/examples/client/multi-image/README.md @@ -0,0 +1,2 @@ +# Large scale containerized project scan automation +## diff --git a/examples/client/multi-image/generate-clone.sh b/examples/client/multi-image/generate-clone.sh new file mode 100644 index 00000000..d577d216 --- /dev/null +++ b/examples/client/multi-image/generate-clone.sh @@ -0,0 +1,23 @@ +# Generate baseline project +# +# Project - container mapping + +SUBPROJECTS="\ +dashboard-ui:testcontainer:2.4,\ +docs-ui:testcontainer:2.4,\ +le:testcontainer:2.4,\ +login-app-ui:testcontainer:2.4,\ +nlv:testcontainer:2.4" + +# SPECFILE=~/Documents/Ciena/excelparameters/BP_SampleProduct.xlsx +SPECFILE=~/Documents/Ciena/excelparameters/BP_SampleProduct_Truncated.xlsx +TEXTFILE=~/Documents/Ciena/excelparameters/bdscaninput.txt + +ls -l $SPECFILE + +COMMAND="python3 examples/client/multi-image/manage_project_structure.py" + +# $COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -sp $SUBPROJECTS --clone-from 2.3 $@ +$COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $SPECFILE --clone-from 2.3 $@ +# $COMMAND -u $BD_URL -t <(echo $API_TOKEN) -nv -pg "Test Group" -p P3 -pv 2.4 -ssf $TEXTFILE --clone-from 2.3 $@ + diff --git a/examples/client/multi-image/manage_project_structure.py b/examples/client/multi-image/manage_project_structure.py new file mode 100644 index 00000000..39b311e1 --- /dev/null +++ b/examples/client/multi-image/manage_project_structure.py @@ -0,0 +1,574 @@ +#!/usr/bin/env python3 +''' + +Copyright (C) 2023 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +''' + +program_description = \ +''' +This script will scan a multi-container project into +hierarchical structure. + +Project Name + Project Version + Sub-project Name + Sub-project Version + Base Image + Base Image Version + Add on Image + Add on Image Version + +Sub-projects are specified as sub-project:[container]:[tag] +if container name omitted it will be set to sub-project +if tag omitted it would be set to 'latest' + +Sub-projects an be specified in excel file with -ssf --subproject-spec-file parameter. +Excel file should contain one worksheet with first row containing column names as following: +Container Name, Image ID, Version, Project Name +and subsequent rows containing data + +Sub-projects could be specified in a text file with -ssf --subproject-spec-file parameter +Each line will have to contain full image specification. +Specification will be parsed and image name prefixed by -str parameter will be used as a sub-project name +In this mode any image that is not residing on //your domain for example abc.com// repository will be skipped. +If -str parameter is empty, Project Name will be used instead. + +Container image name scanned will be written into project version nickname field + +''' + +import argparse +import json +import logging +import sys +import arrow + +from io import StringIO + +from blackduck import Client +from pprint import pprint,pformat + +class MultiImageProjectManager(): + + def __init__(self, args): + self.debug = args.debug + self.binary = args.binary + self.individual_file_matching = args.individual_file_matching + self.log_config() + self.base_url = args.base_url + with open(args.token_file, 'r') as tf: + self.access_token = tf.readline().strip() + self.no_verify = args.no_verify + self.reprocess_run_file = args.reprocess_run_file + self.connect() + if self.reprocess_run_file: + self.load_project_data() + else: + self.init_project_data(args) + self.serialize = args.serialize + self.skip_group=args.skip_group + + def connect(self): + self.client = Client(base_url=self.base_url, token=self.access_token, verify=self.no_verify, timeout=60.0, retries=4) + + def load_project_data(self): + with open(self.reprocess_run_file, "r") as f: + data = json.load(f) + self.project_data = data + self.project_data.pop('log', None) + discard = list() + for name, subproject in self.project_data['subprojects'].items(): + if not self.has_errors(subproject): + discard.append(name) + else: + self.project_data['subprojects'][name].pop('log', None) + self.project_data['subprojects'][name].pop('status', None) + self.project_data['subprojects'][name].pop('scan_results',None) + for name in discard: + del self.project_data['subprojects'][name] + + def has_errors(self, subproject): + structure = False + runtime = False + if subproject['status'] != 'PRESENT': + structure = True + if not subproject.get('scan_results', None): + runtime = True + else: + rcodes = [r['scan_results']['returncode'] for r in subproject['scan_results'] if r.get('scan_results', None)] + if sum(rcodes) > 0: + runtime = True + if structure or runtime: + return True + else: + return False + + def init_project_data(self,args): + self.project_data = dict() + self.project_data['project_name'] = args.project_name + self.project_data['version_name'] = args.version_name + self.project_data['project_group'] = args.project_group + self.project_data['clone_from'] = args.clone_from + self.project_data['dry_run'] = args.dry_run + self.project_data['strict'] = args.strict + self.project_data['remove'] = args.remove + if args.remove: + return + + subprojects = dict() + child_spec_list = self.get_child_spec_list(args) + # pprint (child_spec_list) + for child_spec in [x.split(':') for x in child_spec_list]: + subproject = dict() + i = iter(child_spec) + i = iter(child_spec) + child = next(i) + repo = next(i, child) + tag = next(i,'latest') + container_spec = f"{repo}:{tag}" + while child in subprojects: + child += "_" + subproject['image'] = container_spec + subproject['project_name'] = child + subproject['version_name'] = args.version_name + subproject['project_group'] = args.project_group + subproject['clone_from'] = args.clone_from + subprojects[child] = subproject + + self.project_data['subprojects'] = subprojects + + def log_config(self): + if self.debug: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.DEBUG) + else: + logging.basicConfig(format='%(asctime)s:%(levelname)s:%(module)s: %(message)s', stream=sys.stderr, level=logging.INFO) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("blackduck").setLevel(logging.WARNING) + + def log(self, level, msg, target=None): + if target: + target_log = target.get('log', None) + if not target_log: + target_log = list() + target['log'] = target_log + target_log.append(msg) + log_function = getattr(logging, level) + log_function(msg) + + def remove_project_structure(self, project_name, version_name): + project = self.find_project_by_name(project_name) + + if not project: + logging.debug(f"Project {project_name} does not exist.") + return + num_versions = self.client.get_resource('versions', project, items=False)['totalCount'] + version = self.find_project_version_by_name(project,version_name) + if not version: + logging.debug(f"Project {project_name} with version {version_name} does not exist.") + return + components = [ + c for c in self.client.get_resource('components',version) if c['componentType'] == "SUB_PROJECT" + ] + logging.debug(f"Project {project_name}:{version_name} has {len(components)} subprojects") + for component in components: + component_name = component['componentName'] + component_version_name = component['componentVersionName'] + logging.debug(f"Removing subproject {component_name} from {project_name}:{version_name}") + component_url = component['_meta']['href'] + response = self.client.session.delete(component_url) + logging.debug(f"Operation completed with {response}") + self.remove_project_structure(component_name, component_version_name) + logging.debug(f"Removing {project_name}:{version_name}") + if num_versions > 1: + response = self.client.session.delete(version['_meta']['href']) + else: + response = self.client.session.delete(project['_meta']['href']) + logging.debug(f"Operation completed with {response}") + + def remove_codelocations_recursively(self, version): + components = self.client.get_resource('components', version) + subprojects = [x for x in components if x['componentType'] == 'SUB_PROJECT'] + logging.debug(f"Found {len(subprojects)} subprojects") + self.unmap_all_codelocations(version) + for subproject in subprojects: + subproject_name = subproject['componentName'] + subproject_version_name = subproject['componentVersionName'] + project = self.find_project_by_name(subproject_name) + if not project: + logging.debug(f"Project {subproject_name} does not exist.") + return + subproject_version = self.find_project_version_by_name(project, subproject_version_name) + if not subproject_version: + logging.debug(f"Project {subproject_name} with version {subproject_version_name} does not exist.") + return + self.remove_codelocations_recursively(subproject_version) + + def unmap_all_codelocations(self, version): + codelocations = self.client.get_resource('codelocations',version) + for codelocation in codelocations: + logging.debug(f"Un-mapping of code location {codelocation['name']}") + codelocation['mappedProjectVersion'] = "" + response = self.client.session.put(codelocation['_meta']['href'], json=codelocation) + logging.debug(f"Un-mapping of code location {codelocation['name']} completed with {response}") + + + def find_or_create_project_group(self, group_name): + url = '/api/project-groups' + params = { + 'q': [f"name:{group_name}"] + } + groups = [p for p in self.client.get_items(url, params=params) if p['name'] == group_name] + if len(groups) == 0: + headers = { + 'Accept': 'application/vnd.blackducksoftware.project-detail-5+json', + 'Content-Type': 'application/vnd.blackducksoftware.project-detail-5+json' + } + data = { + 'name': group_name + } + response = self.client.session.post(url, headers=headers, json=data) + return response.headers['Location'] + else: + return groups[0]['_meta']['href'] + + def find_project_by_name(self, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in self.client.get_resource('projects', params=params) if p['name'] == project_name] + if len(projects) == 1: + return projects[0] + else: + return None + + def find_project_version_by_name(self, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in self.client.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + + def create_project_version(self,project_name,version_name,nickname = None): + version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name} + if nickname: + version_data['nickname'] = nickname + url = '/api/projects' + project = self.find_project_by_name(project_name) + if project: + data = version_data + url = project['_meta']['href'] + '/versions' + else: + data = {"name": project_name, + "projectGroup": self.find_or_create_project_group(self.project_data['project_group']), + "versionRequest": version_data} + return self.client.session.post(url, json=data) + + def add_component_to_version_bom(child_version, version): + url = version['_meta']['href'] + '/components' + data = { 'component': child_version['_meta']['href']} + return self.client.session.post(url, json=data) + + def process_excel_spec_file(self,wb): + ws = wb.active + project_list = [] + row_number = 0 + for row in ws.values: + row_number += 1 + if (row_number == 1 and + row[0] == 'Container Name' and + row[1] == 'Image ID' and + row[2] == 'Version' and + row[3] == 'Project Name'): + logging.info(f"File Format checks out (kind of)") + continue + elif row_number > 1: + project_list.append(f"{row[3]}:{row[0]}:{row[2]}") + else: + logging.error(f"Could not parse input file ") + sys.exit(1) + return (project_list) + + def process_text_spec_file(self,args): + project_list = [] + prefix = args.string_to_put_in_front_of_subproject_name + if not prefix: + prefix = args.project_name + if args.subproject_spec_file is not None : + with open(args.subproject_spec_file, "r") as f: + lines = f.read().splitlines() + for line in lines: + #print(line) + image_name = line.split('/')[-1].split(':')[0] # Don't look at me, you wrote it! + sub_project_name = "_".join((prefix, image_name)) + spec_line = ":".join((sub_project_name, line)) + # if "//your domain ex. abc.com //" in spec_line: + project_list.append(spec_line) + return (project_list) + + def get_child_spec_list(self,args): + if args.subproject_list: + return args.subproject_list.split(',') + else: + # Excel and plaintext + logging.debug(f"Processing excel file {args.subproject_spec_file}") + import openpyxl + try: + wb = openpyxl.load_workbook(args.subproject_spec_file) + return self.process_excel_spec_file(wb) + except Exception: + return self.process_text_spec_file(args) + + def create_and_add_child_projects(self,version): + version_url = version['_meta']['href'] + '/components' + version_name = self.project_data['version_name'] + for child_name, child in self.project_data['subprojects'].items(): + child_name = child['project_name'] + container_spec = child['image'] + project = self.find_project_by_name(child_name) + if project: + version = self.find_project_version_by_name(project,version_name) + if version: + if self.project_data['strict']: + self.log('error', f"Child project {project['name']} with version {version_name} exists.", child) + sys.exit(1) + else: + self.log('debug',f"Child project {project['name']} with version {version_name} found.", child) + self.log('debug',f"Recursively removing codelocations for {project['name']} with version {version_name} ", child) + try: + self.log('debug',f"Adding project {child_name} {version_name} to the parent project", child) + child_version_url = version['_meta']['href'] + response = self.client.session.post(version_url,json={'component': child_version_url}) + self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with {response}", child) + except Exception as e: + self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with exception {e}", child) + self.remove_codelocations_recursively(version) + else: + response = self.create_project_version(child_name,version_name, nickname=container_spec) + self.log('debug',f"Creating project {child_name} : {version_name} completed with {response}", child) + if response.ok: + child_version = self.find_project_version_by_name(self.find_project_by_name(child_name),version_name) + child_version_url = child_version['_meta']['href'] + response = self.client.session.post(version_url,json={'component': child_version_url}) + self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with {response}", child) + else: + response = self.create_project_version(child_name,version_name, nickname=container_spec) + self.log('debug',f"Creating project {child_name} : {version_name} completed with {response}",child) + if response.ok: + child_version = self.find_project_version_by_name(self.find_project_by_name(child_name),version_name) + child_version_url = child_version['_meta']['href'] + response = self.client.session.post(version_url,json={'component': child_version_url}) + self.log('debug',f"Adding {child_name} : {version_name} to parent project completed with {response}", child) + + def create_project_structure(self): + project_name = self.project_data['project_name'] + version_name = self.project_data['version_name'] + + project = self.find_project_by_name(project_name) + if project: + self.log('debug', f"Project {project_name} located", self.project_data) + version = self.find_project_version_by_name(project,version_name) + if version: + if self.project_data['strict']: + self.log('error', f"Project {project['name']} with version {version_name} exists.", self.project_data) + sys.exit(1) + else: + self.log('debug',f"Found Project {project['name']} with version {version_name}.", self.project_data) + else: + response = self.create_project_version(project_name,version_name) + if response.ok: + version = self.find_project_version_by_name(self.find_project_by_name(project_name),version_name) + self.log('debug', f"Project {project_name} : {version_name} created", self.project_data) + else: + self.log('debug',f"Failed to create Project {project_name} : {version_name} created", self.project_data) + sys.exit(1) + else: + self.log('debug',f"Project {project_name} was not found, creating ...", self.project_data) + response = self.create_project_version(project_name,version_name) + if response.ok: + version = self.find_project_version_by_name(self.find_project_by_name(project_name),version_name) + self.log('debug',f"Project {project_name} : {version_name} created", self.project_data) + else: + self.log('debug',f"Failed to create Project {project_name} : {version_name} created", self.project_data) + sys.exit(1) + self.log('debug',f"Checking/Adding subprojects to {project_name} : {version['versionName']}", self.project_data) + self.create_and_add_child_projects(version) + + def validate_project_structure(self): + parent_project = self.find_project_by_name(self.project_data['project_name']) + parent_version = self.find_project_version_by_name(parent_project, self.project_data['version_name']) + components = [c for c in self.client.get_resource('components', parent_version) if c['componentType'] == 'SUB_PROJECT'] + for spn, sp in self.project_data['subprojects'].items(): + spn = sp['project_name'] + spvn = sp['version_name'] + ca = [c for c in components if spn == c['componentName'] and spvn == c['componentVersionName']] + if len(ca) == 1: + sp['status'] = 'PRESENT' + self.log('debug',f"Sub-project project {spn} : {spvn} is present in the BOM ", sp) + else: + sp['status'] = 'ABSENT' + self.log('debug',f"Sub-project project {spn} : {spvn} could not be added to the BOM ", sp) + + def scan_container_images(self): + from scan_docker_image_lite import scan_container_image + from blackduck.HubRestApi import HubInstance + hub = HubInstance(self.base_url, api_token=self.access_token, insecure= not self.no_verify, debug=self.debug) + + for child_name, child in self.project_data['subprojects'].items(): + parent_project = child['project_name'] + parent_version = child['version_name'] + image_name = child['image'] + clone_from = child['clone_from'] + project_group = child['project_group'] + detect_options = (f"--detect.parent.project.name={parent_project} " + f"--detect.parent.project.version.name={parent_version} " + f"--detect.project.version.nickname={image_name}") + if self.serialize: + detect_options += f" --detect.wait.for.results=true" + if self.individual_file_matching: + detect_options += f" --detect.blackduck.signature.scanner.individual.file.matching=ALL" + if clone_from: + detect_options += f" --detect.clone.project.version.name={clone_from}" + if project_group: + detect_options += f" --detect.project.group.name=\"{project_group}\"" + try: + results = scan_container_image( + image_name, + None, + None, + None, + parent_project, + parent_version, + detect_options, + hub=hub, + binary=self.binary, + skip_group = self.skip_group + ) + child['scan_results'] = results + except Exception as e: + # import traceback + # traceback.print_exc() + self.log('error', repr(e), child) + logging.error(f"Scanning of {image_name} failed, skipping") + + def proceed(self): + if self.project_data['remove']: + project_name = self.project_data['project_name'] + version_name = self.project_data['version_name'] + self.remove_project_structure(project_name, version_name) + else: + self.create_project_structure() + self.validate_project_structure() + self.scan_container_images() + +def write_failure_report(data, output_file_name): + s = StringIO() + subprojects = data['subprojects'] + for subproject_name, subproject in subprojects.items(): + structure = False + runtime = False + if subproject['status'] != 'PRESENT': + structure = True + if not subproject.get('scan_results', None): + runtime = True + else: + rcodes = [r['scan_results']['returncode'] for r in subproject['scan_results'] if r.get('scan_results', None)] + if sum(rcodes) > 0: + runtime = True + if structure or runtime: + print (f"\nStatus for {subproject['project_name']} {subproject['version_name']}", file = s) + print (f"\tStructural failures present {structure}", file = s) + print (f"\t Runtime failures present {runtime}\n", file = s) + + if subproject['status'] != 'PRESENT': + for line in subproject['log']: + print ('\t', line, file = s) + scan_results = subproject.get('scan_results',[]) + if len(scan_results) == 0: + print ("No scans were performed", file = s) + else: + for invocation in scan_results: + returncode = invocation['scan_results']['returncode'] + if returncode > 0: + print (f"\n\tScan for {invocation['name']} failed with returncode {returncode}\n", file = s) + stdout = invocation['scan_results']['stdout'].split('\n') + for line in stdout: + if 'ERROR' in line and 'certificates' not in line: + print ('\t', line, file = s) + with open(output_file_name, "w") as f: + f.write(s.getvalue()) + +def parse_command_args(): + + parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-pg", "--project_group", required=False, default='Multi-Image', help="Project Group to be used") + parser.add_argument("-p", "--project-name", required=False, help="Project Name") + parser.add_argument("-pv", "--version-name", required=False, help="Project Version Name") + group = parser.add_mutually_exclusive_group() + group.add_argument("-sp", "--subproject-list", required=False, help="List of subprojects to generate with subproject:container:tag") + group.add_argument("-ssf", "--subproject-spec-file", required=False, help="Excel or txt file containing subproject specification") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-rm", "--remove", action='store_true', required=False, help="Remove project structure with all subprojects (DANGEROUS!)") + parser.add_argument("--clone-from", required=False, help="Main project version to use as template for cloning") + parser.add_argument("--dry-run", action='store_true', required=False, help="Create structure only, do not execute scans") + parser.add_argument("-str", "--string-to-put-in-front-of-subproject-name", required=False, help="Prefix string for subproject names" ) + parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on") + parser.add_argument("--strict", action='store_true', help="Fail if existing (sub)project versions already exist") + parser.add_argument("--binary", action='store_true', help="Use binary scan for analysis") + parser.add_argument("-ifm", "--individual-file-matching", action='store_true', help="Turn Individual file matching on") + parser.add_argument("--reprocess-run-file", help="Reprocess Failures from previous run report.") + parser.add_argument("--serialize", action='store_true', help="Serialize scan submissions by adding --detect.wait.for.results=true to scan invocations") + parser.add_argument("--skip-group", required=False, help="exclude layers belog to specific groups, ex. 'base' ") + args = parser.parse_args() + if not args.reprocess_run_file and not (args.project_name and args.version_name): + parser.error("[ -p/--project-name and -pv/--version-name ] or --reprocess-run-file are required") + if args.reprocess_run_file and (args.project_name or args.version_name): + parser.error("[ -p/--project-name and -pv/--version-name ] or --reprocess-run-file are required") + return args + +def main(): + from datetime import datetime + timestamp = datetime.now().strftime("%Y-%m-%d_%H%M%S") + args = parse_command_args() + mipm = MultiImageProjectManager(args) + if not args.remove: + logging.info(f"Parsed {len(mipm.project_data['subprojects'])} projects from specification data") + mipm.proceed() + + if not args.remove: + filename_base = f"{mipm.project_data['project_name']}-{mipm.project_data['version_name']}" + filename_complete = f"{filename_base}-{timestamp}-full.json" + filename_failure_report = f"{filename_base}-{timestamp}-failures.txt" + # write full processing log + with open (filename_complete, "w") as f: + json.dump(mipm.project_data, f, indent=2) + + write_failure_report(mipm.project_data, filename_failure_report) + +if __name__ == "__main__": + sys.exit(main()) + + diff --git a/examples/client/multi-image/scan_docker_image_lite.py b/examples/client/multi-image/scan_docker_image_lite.py new file mode 100644 index 00000000..a771588b --- /dev/null +++ b/examples/client/multi-image/scan_docker_image_lite.py @@ -0,0 +1,572 @@ +''' +Created on June 7, 2023 +@author: kumykov + +Alternative version if Docker image layer by layer scan. + +This program will download docker image and scan it into Black Duck server layer by layer +Each layer will be scanned as a separate scan with a signature scan. + +Layers in the container images could be grouped into groups of contiguous layers. + +I.e. +layers 1-5 - Group 1, layers 6-8 - Group 2, etc. + +Each group will be scanned as a version within a project + +Project naming will follow docker container image specification + + repository/image-name:version + +Will create project named "repository/image-name" and will have "version" as a version prefix + +Project versions corresponding to groups will be named + + version_group_name + +Scans will be named as + + repository/image-name_version_layer_1 + repository/image-name_version_layer_2 + ......... + +Layers are numbered in chronological order + +If a dockerfile or a base image spec is available, grouping could be done based on the +information gathered from those sources. + +layers that are present in the base image will be grouped as *_base_* +layers that are not present in the base image will be grouped as *_addon_* + + + +Usage: + +scan_docker_image_slim.py [-h] imagespec [--grouping=group_end:group_name,group_end:group_name] | [--dockerfile=Dockerfile | --base-image=baseimagespec] + +positional arguments: + imagespec Container image tag, e.g. repository/imagename:version + +optional arguments: + -h, --help show this help message and exit + --grouping GROUPING Group layers into user defined project versions (can't be used with --base-image) + --base-image BASE_IMAGE + Use base image spec to determine base image/layers (can't be used with --grouping or + --dockerfile) + --dockerfile DOCKERFILE + Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image) + --project-name Specify project name (default is container image spec) + --project-version Specify project version (default is container image tag/version) + --detect-options DETECT_OPTIONS + Extra detect options to be passed directly to the detect + + +Using --detect-options + +It is possible to pass detect options directly to detect command. +For example one wants to specify cloning options directly + +python3 scan_docker_image_lite.py --detect-options='--detect.clone.project.version.name=version --detect.project.clone.categories=COMPONENT_DATA,VULN_DATA' + +There is not validation of extra parameters passed, use with care. + +MK 2022-10-24 Automating grouping. +By adding specific markers to the Dockerfile, it is possible to enable automating group detection. + +In the Dockerfile, once the specific group is complete, add the following command: + +RUN echo _group_end + +e.g. + +. . . +RUN echo base_group_end +. . . +RUN echo app_group_end +. . . + +that will generate grouping as N:base,N:app + + +''' + +from blackduck.HubRestApi import HubInstance +from pprint import pprint +from sys import argv +import json +import logging +import os +import requests +import shutil +import subprocess +import sys +from argparse import ArgumentParser +import argparse +import re + +#hub = HubInstance() + +''' +quick and dirty wrapper to process some docker functionality +''' +class DockerWrapper(): + + def __init__(self, workdir, scratch = True): + self.workdir = workdir + self.imagedir = self.workdir + "/container" + self.imagefile = self.workdir + "/image.tar" + if scratch: + self.initdir() + self.docker_path = self.locate_docker() + + def initdir(self): + if os.path.exists(self.workdir): + if os.path.isdir(self.workdir): + shutil.rmtree(self.workdir) + else: + os.remove(self.workdir) + os.makedirs(self.workdir, 0o755, True) + os.makedirs(self.workdir + "/container", 0o755, True) + + + def locate_docker(self): + os.environ['PATH'] += os.pathsep + '/usr/local/bin' + args = [] + args.append('/usr/bin/which') + args.append('docker') + proc = subprocess.Popen(['which','docker'], stdout=subprocess.PIPE) + out, err = proc.communicate() + lines = out.decode().split('\n') + if 'docker' in lines[0]: + return lines[0] + else: + raise Exception('Can not find docker executable in PATH.') + + def pull_container_image(self, image_name): + args = [] + args.append(self.docker_path) + args.append('pull') + args.append(image_name) + return subprocess.run(args, capture_output=True) + + def get_container_image_history(self, image_name): + args = [] + args.append(self.docker_path) + args.append('history') + args.append(image_name) + result = subprocess.run(args, capture_output=True) + return result + + def save_container_image(self, image_name): + args = [] + args.append(self.docker_path) + args.append('save') + args.append('-o') + args.append(self.imagefile) + args.append(image_name) + return subprocess.run(args) + + def unravel_container(self): + args = [] + args.append('tar') + args.append('xvf') + args.append(self.imagefile) + args.append('-C') + args.append(self.imagedir) + return subprocess.run(args, capture_output=True) + + def read_manifest(self): + filename = self.imagedir + "/manifest.json" + with open(filename) as fp: + data = json.load(fp) + return data + + def read_config(self): + manifest = self.read_manifest() + configFile = self.imagedir + "/" + manifest[0]['Config'] + with open(configFile) as fp: + data = json.load(fp) + return data + + def read_oci_layout(self): + oci_layout_file = self.imagedir + "/" + 'oci-layout' + if os.path.exists(oci_layout_file) and os.path.isfile(oci_layout_file): + with open(oci_layout_file) as fp: + data = json.load(fp) + return data + else: + return None + +class Detector(): + def __init__(self, hub): + # self.detecturl = 'https://blackducksoftware.github.io/hub-detect/hub-detect.sh' + # self.detecturl = 'https://detect.synopsys.com/detect.sh' + # self.detecturl = 'https://detect.synopsys.com/detect7.sh' + # self.detecturl = 'https://detect.synopsys.com/detect8.sh' + # self.detecturl = 'https://detect.blackduck.com/detect9.sh' + self.detecturl = 'https://detect.blackduck.com/detect10.sh' + self.baseurl = hub.config['baseurl'] + self.filename = '/tmp/hub-detect.sh' + self.token=hub.config['api_token'] + self.baseurl=hub.config['baseurl'] + self.download_detect() + + def download_detect(self): + with open(self.filename, "wb") as file: + response = requests.get(self.detecturl) + file.write(response.content) + + def detect_run(self, options=['--help']): + cmd = ['bash'] + cmd.append(self.filename) + cmd.append('--blackduck.url=%s' % self.baseurl) + cmd.append('--blackduck.api.token=' + self.token) + cmd.append('--blackduck.trust.cert=true') + cmd.extend(options) + return subprocess.run(cmd, capture_output=True) + +class ContainerImageScanner(): + + def __init__( + self, hub, container_image_name, workdir='/tmp/workdir', + grouping=None, base_image=None, dockerfile=None, detect_options=None, skip_group=None): + self.hub = hub + self.hub_detect = Detector(hub) + self.docker = DockerWrapper(workdir) + self.container_image_name = container_image_name + cindex = container_image_name.rfind(':') + if cindex == -1: + self.image_name = container_image_name + self.image_version = 'latest' + else: + self.image_name = container_image_name[:cindex] + self.image_version = container_image_name[cindex+1:] + self.grouping = grouping + self.base_image = base_image + self.dockerfile = dockerfile + self.base_layers = None + self.project_name = self.image_name + self.project_version = self.image_version + self.extra_options = [] + if detect_options: + self.extra_options = detect_options.split(" ") + self.binary = False + if skip_group: + self.skip_group=skip_group.split(",") + else: + self.skip_group=[] + + + def prepare_container_image(self): + self.docker.initdir() + result = self.docker.pull_container_image(self.container_image_name) + logging.debug(f"Command {' '.join(result.args)} exited with returncode {result.returncode}") + result = self.docker.save_container_image(self.container_image_name) + if result.returncode: + raise Exception (f"Command {' '.join(result.args)} failed with returncode {result.returncode} error = {result.stdout}") + result = self.docker.unravel_container() + if result.returncode: + raise Exception (f"Command {' '.join(result.args)} failed with returncode {result.returncode} error = {result.stdout}") + # result = self.docker.get_container_image_history(self.container_image_name) + history = self.docker.read_config()['history'] + layer_count = 0 + history_grouping = '' + for item in history: + if not item.get('empty_layer', None): + layer_count += 1 + match = re.search('echo (.+?)_group_end', item.get('created_by','')) + if match: + found = match.group(1) + if len(history_grouping): + history_grouping += ',' + history_grouping += str(layer_count) + ":" + found + if len(history_grouping) and self.grouping == '1024:everything': + self.grouping = history_grouping + self.oci_layout = self.docker.read_oci_layout() + + def process_container_image_by_user_defined_groups(self): + self.manifest = self.docker.read_manifest() + self.config = self.docker.read_config() + + if self.grouping: + self.groups = dict(x.split(":") for x in self.grouping.split(",")) + + self.layers = [] + num = 1 + offset = 0 + for i in self.manifest[0]['Layers']: + layer = {} + if self.grouping: + intlist = [int(x) for x in sorted(self.groups.keys())] + intlist.sort() + key_number = len(self.groups) - len([i for i in intlist if i >= num]) + if key_number >= len(self.groups): + layer['group_name'] = "undefined" + else: + layer['group_name'] = self.groups.get(str(intlist[key_number])) + layer['project_version'] = self.project_version + layer['name'] = "{}_{}_{}_layer_{}".format(self.project_name,self.project_version,layer['group_name'],str(num)) + layer['project_name'] = "{}_{}".format(self.project_name,layer['group_name']) + else: + layer['project_version'] = self.project_version + layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(num) + layer['project_name'] = self.project_name + layer['path'] = i + while self.config['history'][num + offset -1].get('empty_layer', False): + offset = offset + 1 + layer['command'] = self.config['history'][num + offset - 1] + layer['shaid'] = self.config['rootfs']['diff_ids'][num - 1] + self.layers.append(layer) + num = num + 1 + # print (json.dumps(self.layers, indent=4)) + + def process_container_image_by_base_image_info(self): + self.manifest = self.docker.read_manifest() + self.config = self.docker.read_config() + + self.layers = [] + num = 1 + offset = 0 + for i in self.manifest[0]['Layers']: + layer = {} + layer['project_name'] = self.project_name + layer['path'] = i + while self.config['history'][num + offset -1].get('empty_layer', False): + offset = offset + 1 + layer['command'] = self.config['history'][num + offset - 1] + layer['shaid'] = self.config['rootfs']['diff_ids'][num - 1] + + if self.base_layers: + pass + if layer['shaid'] in self.base_layers: + layer['group_name'] = 'base' + else: + layer['group_name'] = 'addon' + layer['project_version'] = "{}_{}".format(self.project_version,layer['group_name']) + layer['name'] = "{}_{}_{}_layer_{}".format(self.project_name,self.project_version,layer['group_name'],str(num)) + else: + layer['project_version'] = self.project_version + layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(num) + self.layers.append(layer) + num = num + 1 + #print (json.dumps(self.layers, indent=4)) + + def process_oci_container_image_by_user_defined_groups(self): + self.manifest = self.docker.read_manifest() + self.config = self.docker.read_config() + + self.layers = self.config['history'] + tagged_layers = [x for x in self.layers if '_group_end' in x.get('created_by','')] + groups = {re.search('echo (.+?)_group_end', str(x.get('created_by',''))).group(1): self.layers.index(x) for x in tagged_layers} + logging.debug(f"Container configuration defines following groups {groups}") + layer_paths = self.manifest[0]['Layers'].copy() + empty_layers = [x for x in self.layers if x.get('empty_layer', False)] + logging.debug(f"Total layers: {len(self.layers)} total paths: {len(layer_paths)} empty layers: {len(empty_layers)}") + + assert len(self.layers) == len(layer_paths) + len(empty_layers), "Something is wrong with this image, Layer math does not add up." + + for layer in self.layers: + layer['index'] = self.layers.index(layer) + if self.grouping: + layer['group_name'] = self.get_group_name(groups, layer['index']) + layer['project_name'] = "{}_{}".format(self.project_name,layer['group_name']) + layer['project_version'] = self.project_version + layer['name'] = "{}_{}_{}_layer_{}".format(self.project_name,self.project_version,layer['group_name'],str(layer['index'])) + else: + layer['project_name'] = self.project_name + layer['project_version'] = self.project_version + layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(layer['index']) + if not layer.get('empty_layer', False): + layer['path'] = layer_paths.pop(0) + #print (json.dumps(self.layers, indent=4)) + + def get_group_name(self, groups, index): + group_name = 'undefined' + for group, value in groups.items(): + if index <= value: + group_name = group + break + return group_name + + def process_oci_container_image_by_base_image_info(self): + print ("Processing by Base Image not supported for OCI images") + sys.exit(1) + pass + + def process_container_image(self): + if self.oci_layout: + self.process_oci_container_image() + else: + self.process_docker_container_image() + + def process_docker_container_image(self): + if self.grouping: + self.process_container_image_by_user_defined_groups() + else: + self.process_container_image_by_base_image_info() + + def process_oci_container_image(self): + if self.grouping: + self.process_oci_container_image_by_user_defined_groups() + else: + self.process_oci_container_image_by_base_image_info() + + def submit_layer_scans(self): + for layer in self.layers: + + #print(f"layer group name={layer['group_name']} skip_group ={self.skip_group}") + + if layer['group_name'] not in self.skip_group: + + if not layer.get('empty_layer', False): + options = [] + options.append('--detect.project.name={}'.format(layer['project_name'])) + options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) + options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) + if self.binary: + options.append('--detect.tools=BINARY_SCAN') + options.append('--detect.binary.scan.file.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.tools=SIGNATURE_SCAN') + if self.oci_layout: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) + if self.base_image or self.grouping or self.dockerfile: + options.extend(self.adorn_extra_options(layer)) + else: + options.extend(self.extra_options) + logging.debug(f"Submitting scan for {layer['name']}") + completed = self.hub_detect.detect_run(options) + scan_results = dict() + for key, value in vars(completed).items(): + if type(value) is bytes: + scan_results[key] = value.decode('utf-8') + else: + scan_results[key] = value + layer['scan_results'] = scan_results + logging.debug(f"Detect run for {layer['name']} completed with returncode {completed.returncode}") + + def adorn_extra_options(self, layer): + result = list() + option_to_adorn = '--detect.clone.project.version.name=' + for option in self.extra_options: + if option.startswith(option_to_adorn): + # result.append(option.rstrip() + "_" + layer['group_name']) + result.append(option.rstrip()) + else: + result.append(option) + return result + + def get_base_layers(self): + if (not self.dockerfile)and (not self.base_image): + raise Exception ("No dockerfile or base image specified") + imagelist = [] + + if self.dockerfile: + from pathlib import Path + dfile = Path(self.dockerfile) + if not dfile.exists(): + raise Exception ("Dockerfile {} does not exist",format(self.dockerfile)) + if not dfile.is_file(): + raise Exception ("{} is not a file".format(self.dockerfile)) + with open(dfile) as f: + for line in f: + if 'FROM' in line.upper(): + a = line.split() + if a[0].upper() == 'FROM': + imagelist.append(a[1]) + if self.base_image: + imagelist.append(self.base_image) + + # print (imagelist) + base_layers = [] + for image in imagelist: + self.docker.initdir() + self.docker.pull_container_image(image) + self.docker.save_container_image(image) + self.docker.unravel_container() + manifest = self.docker.read_manifest() + # print(manifest) + config = self.docker.read_config() + # print(config) + base_layers.extend(config['rootfs']['diff_ids']) + return base_layers + + +def scan_container_image( + imagespec, grouping=None, base_image=None, dockerfile=None, + project_name=None, project_version=None, detect_options=None, hub=None, binary=False, skip_group=None ): + + if hub: + hub = hub + else: + hub = HubInstance() + scanner = ContainerImageScanner( + hub, imagespec, grouping=grouping, base_image=base_image, + dockerfile=dockerfile, detect_options=detect_options, skip_group=skip_group) + if project_name: + scanner.project_name = project_name + if project_version: + scanner.project_version = project_version + if not grouping: + if not base_image and not dockerfile: + scanner.grouping = '1024:everything' + else: + scanner.base_layers = scanner.get_base_layers() + if binary: + scanner.binary = True + logging.info(f"Scanning image {imagespec}") + + scanner.prepare_container_image() + scanner.process_container_image() + scanner.submit_layer_scans() + return scanner.layers + +def main(argv=None): + + if argv is None: + argv = sys.argv + else: + argv.extend(sys.argv) + + parser = ArgumentParser() + parser.add_argument('imagespec', help="Container image tag, e.g. repository/imagename:version") + parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined project versions (can't be used with --base-image)") + parser.add_argument('--base-image',default=None, type=str, help="Use base image spec to determine base image/layers (can't be used with --grouping or --dockerfile)") + parser.add_argument('--dockerfile',default=None, type=str, help="Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image)") + parser.add_argument('--project-name',default=None, type=str, help="Specify project name (default is container image spec)") + parser.add_argument('--project-version',default=None, type=str, help="Specify project version (default is container image tag/version)") + parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directly to the detect") + parser.add_argument('--binary', action='store_true', help="Use Binary Scan instead of signature scan") + + args = parser.parse_args() + + logging.debug(args); + + if not args.imagespec: + parser.print_help(sys.stdout) + sys.exit(1) + + if args.dockerfile and args.base_image: + parser.print_help(sys.stdout) + sys.exit(1) + + if args.grouping and (args.dockerfile and args.base_image): + parser.print_help(sys.stdout) + sys.exit(1) + + scan_container_image( + args.imagespec, + args.grouping, + args.base_image, + args.dockerfile, + args.project_name, + args.project_version, + args.detect_options, + args.binary) + + +if __name__ == "__main__": + sys.exit(main()) + diff --git a/examples/client/net_add_components.py b/examples/client/net_add_components.py new file mode 100644 index 00000000..eaa5debe --- /dev/null +++ b/examples/client/net_add_components.py @@ -0,0 +1,99 @@ +#$!/usr/bin/env python3 +# + +''' +Created on Sep 23, 2024 +@author: kumykov + +Net Add Components. + +This script will extract components form a source project and add them as +Manually Added components to the target project + +''' +from blackduck import Client + +import argparse +import logging +import sys +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + assert len(projects) == 1, f"Project {project_name} not found." + return projects[0] + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + assert len(versions) == 1, f"Project version {version_name} for project {project['name']} not found" + return versions[0] + +def parse_command_args(): + + parser = argparse.ArgumentParser("Extract components form a source project and add them to the target project as manual components.\n") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument('-sp', '--source-project', help="Source Project") + parser.add_argument('-sv', '--source-version', help="Source Project Version") + parser.add_argument('-tp', '--target-project', help="Target Project") + parser.add_argument('-tv', '--target-version', help="Target Project Version") + + return parser.parse_args() + +def add_component_to_a_project_version(bd, component, components_url): + headers = { + "Content-Type": "application/vnd.blackducksoftware.bill-of-materials-6+json" + } + data = dict() + component_version = component.get('componentVersion', component.get('component')) + component_origins = component['origins'] + component_license = component['licenses'][0]['license'] + if len(component_origins) > 0: + for origin in component_origins: + origin_url = origin['origin'] + payload = {"component": origin_url} + payload['license'] = component_license + result = bd.session.post(components_url, json=payload, headers=headers) + pprint(result) + else: + data['component'] = component_version + data['license'] = component_license + result = bd.session.post(components_url, json=data, headers=headers) + pprint(result) + + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + global bd + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + + project = find_project_by_name(bd, args.source_project) + version = find_project_version_by_name(bd, project, args.source_version) + + target_project = find_project_by_name(bd, args.target_project) + target_version = find_project_version_by_name(bd, target_project, args.target_version) + + dict = bd.list_resources(target_version) + components_url = dict['components'] + + components = bd.get_resource('components', version) + for component in components: + add_component_to_a_project_version(bd, component, components_url) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/client/parse_spdx.py b/examples/client/parse_spdx.py new file mode 100644 index 00000000..c20c01e7 --- /dev/null +++ b/examples/client/parse_spdx.py @@ -0,0 +1,956 @@ +''' +Created on August 15, 2023 +@author: swright + +Copyright (C) 2023 Synopsys, Inc. +http://www.blackducksoftware.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +This script will parse a provided SPDX file and import the SBOM to the +specified Project Name and Project Version. + +Then it will search each component specified in the SPDX file to determine +if the component was succesfully imported. Any missing components will be +added as a custom component and then added to the Project+Verion's BOM. + +All missing components are saved to a file in JSON format for future reference. + +Version History +1.0 2023-09-26 Initial Release +1.1 2023-10-13 Updates to improve component matching of BD Component IDs +1.2 2023-11-03 - Handle BD component with no version + - Fix bug related to extrefs with both purl and BD component data + - Check if project every had a "non-SBOM" scan and exit if so + - Fix some invalid sort parameter formatting + - Limit notification checking to last 24 hours +1.3 2023-11-14 - Force encoding utf8 when opening file + - Use the component URL from API call in the version query - resolves + some situations where the KB version lookup fails + - Update find_comp_in_bom to return the matching URL instead of + True/False + - Track unique BOM matches by tracking the matched component URL + returned by find_comp_in_bom + - Track the count of skipped items from the SPDX + - Make the unique package tracking more accurate - do not include skipped items + - Create fall-through matching. First check BD component, then the purl info + (rather than only checking the purl) +1.4 2023-11-21 - Check the component-import-events API for improved BOM + component searching accuracy + +Requirements +- python3 version 3.8 or newer recommended +- The following packages are used by the script and should be installed + prior to use: + argparse + blackduck + sys + logging + time + json + pprint + spdx_tools + re + pathlib + datetime + +- Blackduck instance +- API token with sufficient privileges + +Install python packages with the following command: + + pip3 install datetime argparse blackduck sys logging time json pprint pathlib spdx_tools + +usage: parse_spdx.py [-h] --base-url BASE_URL --token-file TOKEN_FILE + --spdx-file SPDX_FILE --out-file OUT_FILE --project + PROJECT_NAME --version VERSION_NAME + [--license LICENSE_NAME] [--no-verify] + [--no-spdx-validate] + +Parse SPDX file and verify if component names are in current SBOM for given +project-version + +optional arguments: + -h, --help show this help message and exit + --base-url BASE_URL Hub server URL e.g. https://your.blackduck.url + --token-file TOKEN_FILE + Access token file + --spdx-file SPDX_FILE + SPDX input file + --out-file OUT_FILE Unmatched components file + --project PROJECT_NAME + Project that contains the BOM components + --version VERSION_NAME + Version that contains the BOM components + --license LICENSE_NAME + License name to use for custom components (default: + "NOASSERTION") + --no-verify Disable TLS certificate verification + --no-spdx-validate Disable SPDX validation +''' + +from blackduck import Client +import argparse +import sys +import logging +import time +import json +from datetime import datetime,timedelta,timezone +import re +from pprint import pprint +from pathlib import Path +from spdx_tools.spdx.model.document import Document +from spdx_tools.spdx.validation.document_validator import validate_full_spdx_document +from spdx_tools.spdx.parser.error import SPDXParsingError +from spdx_tools.spdx.parser.parse_anything import parse_file + +# Used when we are polling for successful upload and processing +global MAX_RETRIES +global SLEEP +MAX_RETRIES = 60 +SLEEP = 10 + +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +logging.getLogger("blackduck").setLevel(logging.CRITICAL) + +# Validates BD project and version +# Inputs: +# projname - Name of project +# vername - Name of version +# Returns: Project object, Version object +def get_proj_ver(projname, vername): + # Fetch Project (can only have 1) + params = { + 'q': [f"name:{projname}"] + } + projects = [p for p in bd.get_resource('projects', params=params) + if p['name'] == projname] + assert len(projects) != 0, \ + f"Failed to locate project: {projname}" + assert len(projects) == 1, \ + f"There should one project named {projname}. Found {len(projects)}" + project = projects[0] + + # Fetch Version (can only have 1) + params = { + 'q': [f"versionName:{vername}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) + if v['versionName'] == vername] + assert len(versions) != 0, \ + f"Failed to find project version: {vername}" + assert len(versions) == 1, \ + f"There should be 1 version named {vername}. Found {len(versions)}" + version = versions[0] + + logging.debug(f"Found {project['name']}:{version['versionName']}") + return(project, version) + +# Returns SPDX Document object on success, otherwise exits on parse failure +# Input: file = Filename to process +# Returns: SPDX document object +def spdx_parse(file): + print("Parsing SPDX file...") + start = time.process_time() + try: + document: Document = parse_file(file) + except SPDXParsingError: + logging.exception("Failed to parse spdx file") + sys.exit(1) + + print('SPDX parsing took {:.2f}s'.format(time.process_time() - start)) + return(document) + +# Validates the SPDX file. Logs all validation messages as warnings. +# Input: SPDX document object +def spdx_validate(document): + print("Validating SPDX file...") + start = time.process_time() + validation_messages = validate_full_spdx_document(document) + print('SPDX validation took {:.2f}s'.format(time.process_time() - start)) + + for validation_message in validation_messages: + # Just printing these messages intead of exiting. + # Later when the file is imported, BD errors will be fatal. + logging.warning(validation_message.validation_message) + +# Returns MIME type to provide to scan API +# Input: filename to check +def get_sbom_mime_type(filename): + with open(filename, 'r', encoding="utf8") as f: + data = f.readlines() + content = " ".join(data) + if 'CycloneDX' in content: + return 'application/vnd.cyclonedx' + if 'SPDX' in content: + return 'application/spdx' + return None + +# Poll for notification alerting us of successful BOM computation +# +# Inputs: +# cl_url - Code Loction URL to match +# proj_version_url - Project Version URL to match +# summaries_url - Summaries URL from codelocation +# +# Returns on success. Errors are fatal. +def poll_notifications_for_success(cl_url, proj_version_url, summaries_url): + retries = MAX_RETRIES + sleep_time = SLEEP + + # Limit the query to the last 24 hours (very conservative but also + # keeps us from having to walk thousands of notifications every time) + today=datetime.now().astimezone(timezone.utc) + yesterday=today - timedelta(days=1) + start=yesterday.strftime("%Y-%m-%dT%H:%M:%S.000Z") + params = { + 'filter': ["notificationType:VERSION_BOM_CODE_LOCATION_BOM_COMPUTED"], + 'sort' : ["createdAt DESC"], + 'startDate' : [start] + } + + while (retries): + retries -= 1 + for result in bd.get_items("/api/notifications", params=params): + if 'projectVersion' not in result['content']: + # Shouldn't be possible due to the filter + continue + # We're checking the entire list of notifications, but ours should + # be near the top. + if (result['content']['projectVersion'] == proj_version_url and + result['content']['codeLocation'] == cl_url and + result['content']['scanSummary'] == summaries_url): + print("BOM calculation complete") + return + + print("Waiting for BOM calculation to complete") + # For debugging + #print(f"Searching Notifications for:\n Proj_version: {proj_version_url}\n" + + # f" CodeLocation: {cl_url}\n Summaries: {summaries_url}") + time.sleep(sleep_time) + + logging.error(f"Failed to verify successful BOM computed in {MAX_RETRIES * sleep_time} seconds") + sys.exit(1) + +# Check if this project-version ever had a non-SBOM scan +# If so, we do not want to step on any toes and exit the script before +# attempting any SBOM import. +# Note: Uses an internal API for simplicity +def check_for_existing_scan(projver): + headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} + for source in bd.get_items(f"{projver}/source-trees", headers=headers): + if not re.fullmatch(r".+spdx/sbom$", source['name']): + logging.error(f"Project has a non-SBOM scan. Details:") + pprint(source) + sys.exit(1) + +# Poll for successful scan of SBOM. +# Inputs: +# +# sbom_name: Name of SBOM document (not the filename, the name defined +# inside the json body) +# proj_version_url: Project version url +# +# Returns summaries_url on success (used for processing the import events later) +# Errors will result in fatal exit. +def poll_for_sbom_complete(sbom_name, proj_version_url): + retries = MAX_RETRIES + sleep_time = SLEEP + matched_scan = False + latest_url = None + cl_url = None + + # Replace any spaces in the name with a dash to match BD + sbom_name = sbom_name.replace(' ', '-') + + # Search for the latest scan matching our SBOM name + params = { + 'q': [f"name:{sbom_name}"], + 'sort': ["updatedAt DESC"] + } + + while (retries): + cls = bd.get_resource('codeLocations', params=params) + retries -= 1 + if matched_scan: + # Exit the while() + break + # Save the CL data as we go for debugging + backupcls = [] + for cl in cls: + backupcls.append(cl) + if matched_scan: + # Exit the inner for() + break + print(f"Searching scans for {sbom_name}...") + # Force exact match of: spdx_doc_name + " spdx/sbom" + # BD appends the "spdx/sbom" string to the name. + if cl['name'] != sbom_name + " spdx/sbom": + # No match, keep searching + print(f" {cl['name']} != {sbom_name}" + " spdx/sbom") + continue + + print(" Scan located") + matched_scan = True + cl_url = cl['_meta']['href'] + + print("Checking for latest-scan info...") + for link in (cl['_meta']['links']): + # Locate the scans URL to check for status + if link['rel'] == "latest-scan": + print(" Located latest-scan") + latest_url = link['href'] + break + + # We walked the list of code locations and didn't find a match + if not matched_scan: + print(f" Waiting to locate scan...") + time.sleep(sleep_time) + + if not matched_scan: + logging.error(f"No scan found for SBOM: {sbom_name}") + print("\nCodelocations API data:\n") + pprint(backupcls) + sys.exit(1) + + assert latest_url, "Failed to locate latest-scan reference" + assert cl_url, "Failed to locate codelocation reference" + + # Wait for scanState = SUCCESS + retries = MAX_RETRIES + while (retries): + json_data = bd.get_json(latest_url) + retries -= 1 + if json_data['scanState'] == "SUCCESS": + print("BOM upload complete") + break + elif json_data['scanState'] == "FAILURE": + logging.error(f"SPDX Scan Failure: {json_data['statusMessage']}") + sys.exit(1) + else: + # Only other state should be "STARTED" -- keep polling + print(f"Waiting for scan completion, currently: {json_data['scanState']}") + time.sleep(sleep_time) + + assert json_data, "Failed to locate scanState data" + # If there were ZERO matches, there will never be a notification of + # BOM import success. Short-circuit the check and treat this as success. + if json_data['matchCount'] == 0: + print("No BOM KB matches, continuing...") + return + + # Save the codelocation summaries_url + summaries_url = json_data['_meta']['href'] + + # Check the bom-status endpoint for success + retries = MAX_RETRIES + while (retries): + json_data = bd.get_json(proj_version_url + "/bom-status") + retries -= 1 + if json_data['status'] == "UP_TO_DATE": + print("BOM import complete") + break + elif json_data['status'] == "UP_TO_DATE_WITH_ERRORS" or \ + json_data['status'] == "PROCESSING_WITH_ERRORS": + logging.error(f"BOM Import failure: status is {json_data['status']}") + sys.exit(1) + else: + print(f"Waiting for BOM import completion, current status: {json_data['status']}") + time.sleep(sleep_time) + + if retries == 0: + logging.error(f"Failed to verify successful SBOM import in {retries * sleep_time} seconds") + sys.exit(1) + + # Finally check notifications + poll_notifications_for_success(cl_url, proj_version_url, summaries_url) + + # Any errors above already resulted in fatal exit + return summaries_url + +# Upload provided SBOM file to Black Duck +# Inputs: +# filename - Name of file to upload +# project - Project name to map to +# version - Version name to map to +def upload_sbom_file(filename, project, version): + mime_type = get_sbom_mime_type(filename) + if not mime_type: + logging.error(f"Could not identify file content for {filename}") + sys.exit(1) + files = {"file": (filename, open(filename,"rb"), mime_type)} + fields = {"projectName": project, "versionName": version} + response = bd.session.post("/api/scan/data", files = files, data=fields) + logging.debug(response) + + if response.status_code == 409: + logging.error(f"File {filename} is already mapped to a different project version") + + if not response.ok: + logging.error(f"Failed to upload SPDX file") + try: + pprint(response.json()['errorMessage']) + except: + logging.error(f"Status code: {response.status_code}") + sys.exit(1) + +# Lookup the given pURL in the BD KB. +# +# Inputs: +# extref - pURL to look up +# +# Returns: +# If match: API matching data (the "result" object) +# No match: None +def find_comp_in_kb(extref): + params = { + 'purl': extref + } + for result in bd.get_items("/api/search/kb-purl-component", params=params): + # Should be exactly 1 match when successful + return(result) + + # Fall through -- lookup failed + return(None) + +# Lookup the given BD Compononent Version in the BD KB. +# Note: Match source will be one of: KB, CUSTOM, or KB_MODIFIED +# Any of these should be should be acceptable +# +# Inputs: +# Component UUID +# Component Version UUID +# +# Returns: +# kb_match dictionary that mimics the format returned by find_comp_in_kb: +# keys are: componentName, versionName, version (url of component version) +# No match returns None +def find_comp_id_in_kb(comp, ver): + kb_match = {} + try: + json_data = bd.get_json(f"/api/components/{comp}") + except: + # No component match + return None + kb_match['componentName'] = json_data['name'] + if ver is None: + # Special case where a component was provided but no version. + # Stick the component URL in the version field which we will later use + # to update the BOM. The name of this field is now overloaded but + # reusing it to stay generic. + kb_match['version'] = json_data['_meta']['href'] + kb_match['versionName'] = "UNKNOWN" + return kb_match + + # Update the component url to match the one returned by API + comp_url = json_data['_meta']['href'] + try: + json_data = bd.get_json(f"{comp_url}/versions/{ver}") + except: + # No component version match + return None + kb_match['versionName'] = json_data['versionName'] + + # Add the url of the component-version + kb_match['version'] = json_data['_meta']['href'] + + return kb_match + +# Locate component name + version in component-import-events +# Returns matched name+version on success, None on failure +def find_comp_import_events(match_dict, compname, compver): + key = compname+compver + if key in match_dict: + return match_dict[key] + return None + +# Locate component name + version in BOM +# Inputs: +# compname - Component name to locate +# compver - Component version to locate +# projver - Project version to locate component in BOM +# +# Returns: Component name+version string on success, None on failure +def find_comp_in_bom(compname, compver, projver): + have_match = False + num_match = 0 + + # Lookup existing SBOM for a match + # This is a fuzzy match (see "react" for an example) + params = { + 'q': [f"componentOrVersionName:{compname}"] + } + + # Search BOM for specific component name + comps = bd.get_resource('components', projver, params=params) + for comp in comps: + if comp['componentName'].lower() != compname.lower(): + # The BD API search is inexact. Force our match to be precise. + continue + if compver == "UNKNOWN": + # No version specified in SPDX, so treat it as a match + return comp['componentName']+"NOVERSION" + # Check component name + version name + try: + if comp['componentVersionName'].lower() == compver.lower(): + return comp['componentName']+comp['componentVersionName'] + except: + # Handle situation where it's missing the version name + print(f"comp {compname} in BOM has no version!") + return None + return None + +# Verifies if a custom component and version already exist in the system. +# +# Inputs: +# compname - Component name to locate +# compver - Component version to locate +# Returns: +# CompMatch - Contains matched component url, None for no match +# VerMatch - Contains matched component verison url, None for no match +def find_cust_comp(compname, compver): + params = { + 'q': [f"name:{compname.lower()}"] + } + + matched_comp = None + matched_ver = None + # Warning: Relies on internal header + headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'} + for comp in bd.get_resource('components', params=params, headers=headers): + if compname.lower() == comp['name'].lower(): + # Force exact match + matched_comp = comp['_meta']['href'] + else: + # Keep checking search results + continue + + # Check version + for version in bd.get_resource('versions', comp): + if compver.lower() == version['versionName'].lower(): + # Successfully matched both name and version + matched_ver = version['_meta']['href'] + return(matched_comp, matched_ver) + + # If we got this far, break out of the loop + # We matched the component, but not the version + break + + return(matched_comp, matched_ver) + +# Find URL of license to use for custom compnent creation +# Inputs: +# license_name - Name of license to locate (case-sensitive) +# +# Returns: URL of license successfully matched. Failures are fatal. +def get_license_url(license_name): + params = { + 'q': [f"name:{license_name}"] + } + for result in bd.get_items("/api/licenses", params=params): + # Added precise matching in case of a situation like "NOASSERTION" & "NOASSERTION2" + if (result['name'] == license_name): + return(result['_meta']['href']) + + logging.error(f"Failed to find license {license_name}") + sys.exit(1) + +# Create a custom component. The Name and Version strings are converted to +# lowercase strings to ensure a reliable experience (avoiding dup names +# with varying CapItaliZation) +# +# Inputs: +# name - Name of component to add +# version - Version of component to add +# license - License name +# Returns the URL for the newly created component version URL if successful +def create_cust_comp(name, version, license): + print(f"Adding custom component: {name} {version}") + license_url = get_license_url(license) + data = { + 'name': name.lower(), + 'version' : { + 'versionName' : version, + 'license' : { + 'license' : license_url + }, + } + } + response = bd.session.post("api/components", json=data) + logging.debug(response) + if not response.ok: + # Shouldn't be possible. We checked for existence earlier. + logging.error(response.json()['errors'][0]['errorMessage']) + logging.error(f"Status code: {response.status_code}") + sys.exit(1) + + # Should be guaranteed 1 version because we just created it! + for version in bd.get_items(response.links['versions']['url']): + return(version['_meta']['href']) + +# Create a version for a custom component that already exists. +# Force the version string to be lowercase. +# +# Inputs: +# comp_url - API URL of the component to update +# version - Version to add to existing component +# license - License to use for version +# +# Returns: component version url just created +def create_cust_comp_ver(comp_url, version, license): + license_url = get_license_url(license) + data = { + 'versionName' : version.lower(), + 'license' : { + 'license' : license_url + }, + } + response = bd.session.post(comp_url + "/versions", json=data) + logging.debug(response) + if response.status_code == 412: + # Shouldn't be possible. We checked for existence earlier. + logging.error(f"Version {version} already exists for component") + sys.exit(1) + + if not response.ok: + logging.error(f"Failed to add Version {version} to component") + sys.exit(1) + + return(response.links['self']['url']) + +# Add specified component version url to our project+version SBOM +# Inputs: +# proj_version_url: API URL for a project+version to update +# comp_ver_url: API URL of a component+version to add +# Prints out any errors encountered. Errors are fatal. +def add_to_sbom(proj_version_url, comp_ver_url): + data = { + 'component': comp_ver_url + } + response = bd.session.post(proj_version_url + "/components", json=data) + if not response.ok: + logging.error(response.json()['errors'][0]['errorMessage']) + logging.error(f"Status code: {response.status_code}") + sys.exit(1) + +# Get matched component data from component import events +# Input: Summaries URL +# Output: Dictionary containing components added to BOM +# Key= + +# Value= + +def get_matched_comps(summaries_url): + match_dict = {} # dictionary to be returned + + summary_data = bd.get_json(summaries_url) + links = summary_data['_meta']['links'] + for link in links: + # Locate the component-import-events link + if link['rel'] == "component-import-events": + cie_link = link['href'] + break + + # Only consider successful matches + params = { + 'filter': ["eventName:component_mapping_succeeded"] + } + for comp in bd.get_items(cie_link, params=params): + key = comp['importComponentName']+comp['importComponentVersionName'] + val = comp['componentName']+comp['componentVersionName'] + match_dict[key] = val + + return(match_dict) + + +def parse_command_args(): + parser = argparse.ArgumentParser(description="Parse SPDX file and verify if component names are in current SBOM for given project-version") + parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("--token-file", dest='token_file', required=True,help="Access token file") + parser.add_argument("--spdx-file", dest='spdx_file', required=True, help="SPDX input file") + parser.add_argument("--out-file", dest='out_file', required=True, help="Unmatched components file") + parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components") + parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components") + parser.add_argument("--license", dest='license_name', required=False, default="NOASSERTION", help="License name to use for custom components (default: NOASSERTION)") + parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") + parser.add_argument("--no-spdx-validate", dest='spdx_validate', action='store_false', help="Disable SPDX validation") + return parser.parse_args() + +# Stub to support invocation as a standalone script +# Parses the command-line args, creates a BD object, and inokes import_sbom +def spdx_main_parse_args(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + bdobj = Client(base_url=args.base_url, token=access_token, verify=args.verify) + import_sbom(bdobj, args.project_name, args.version_name, args.spdx_file, \ + args.out_file, args.license_name, args.spdx_validate) + +# Normalize a BD UUID or URL in the extrefs section to be consistently formatted +# Input: Black Duck component or version ID string from SPDX file +# Output: UUID +def normalize_id(id): + # Strip trailing '/' + id = id.rstrip('/') + # Ensure only the UUID remains + id = id.split('/')[-1] + return id + +# Main entry point +# +# Inputs: +# bdobj - BD Client Object +# projname - Name of project +# vername - Name of version +# spdxfile - SPDX file location +# outfile (Optional) - Name of file to write missing component data to in JSON. +# Default: No file written +# license_name - Name of license to use for custom components +# Default: NOASSERTION +# do_spdx_validate - Validate the SPDX file? (Boolean) +# Default: True +def import_sbom(bdobj, projname, vername, spdxfile, outfile=None, \ + license_name="NOASSERTION", do_spdx_validate=True): + + global bd + bd = bdobj + + if (Path(spdxfile).is_file()): + document = spdx_parse(spdxfile) + if (do_spdx_validate): + spdx_validate(document) + else: + logging.error(f"Could not open SPDX file: {spdxfile}") + sys.exit(1) + + # Validate project/version details + project, version = get_proj_ver(projname, vername) + proj_version_url = version['_meta']['href'] + check_for_existing_scan(proj_version_url) + + # Upload the provided SBOM + upload_sbom_file(spdxfile, projname, vername) + + # Wait for scan completion. Will exit if it fails. + summaries_url = poll_for_sbom_complete(document.creation_info.name, proj_version_url) + # Collect the matched component data for later processing + match_dict = get_matched_comps(summaries_url) + + # Open unmatched component file to save name, spdxid, version, and + # origin/purl for later in json format + if outfile: + try: outfile = open(outfile, 'w') + except: + logging.exception("Failed to open file for writing: " + outfile) + sys.exit(1) + + # Stats to track + bom_matches = 0 + kb_matches = 0 + nopurl = 0 + not_in_bom = 0 + cust_added_to_bom = 0 + kb_match_added_to_bom = 0 + package_count = 0 + cust_comp_count = 0 + cust_ver_count = 0 + skip_count = 0 + # Used for tracking repeated package data, not including skips + packages = {} + # Used for tracking unique BOM matches + bom_packages = {} + # Saved component data to write to file + comps_out = [] + + # Walk through each component in the SPDX file + for package in document.packages: + package_count += 1 + # We hope we'll have an external reference (pURL or KBID), but it + # is possible to have neither. + extref = None + + if package.name == "": + # Strange case where the package name is empty. Skip it. + logging.warning("WARNING: Skipping empty package name. Package info:") + skip_count += 1 + pprint(package) + continue + + # Trim any odd leading/trailing space or newlines + package.name = package.name.strip() + + # matchname/matchver can change, depending on the KB lookup step. + # These are stored separately to keep the original names handy + matchname = package.name + if package.version is None or package.version == "": + # Default in case one is not specified in SPDX + package.version = "UNKNOWN" + package.version = package.version.strip() + matchver = package.version + print(f"Processing SPDX package: {matchname} version: {matchver}...") + + kb_match = None + if package.external_references: + # Build dictionary of extrefs for easy access + extrefs = {} + for ref in package.external_references: + # Older BD release prepend this string; strip it + reftype = ref.reference_type.lstrip("LocationRef-") + extrefs[reftype] = ref.locator + + if "BlackDuck-Component" in extrefs: + # Prefer BD component lookup if available + compid = normalize_id(extrefs['BlackDuck-Component']) + try: + verid = normalize_id(extrefs['BlackDuck-ComponentVersion']) + except: + print(" BD Component specified with no version") + verid = None + # Lookup by KB ID + kb_match = find_comp_id_in_kb(compid, verid) + extref = extrefs['BlackDuck-Component'] + if not kb_match: + # BD comp lookup failed, so try purl instead + if "purl" in extrefs: + kb_match = find_comp_in_kb(extrefs['purl']) + extref = extrefs['purl'] + elif "purl" in extrefs: + # If no BD component details are available + kb_match = find_comp_in_kb(extrefs['purl']) + extref = extrefs['purl'] + elif "BlackDuck-Version" in extrefs: + # Skip BD project/versions. These occur in BD-generated BOMs. + skip_count += 1 + print(f" Skipping BD project/version in BOM: {package.name} {package.version}") + continue + else: + nopurl += 1 + print(f" No pURL or KB ID provided for {package.name} {package.version}") + + if (kb_match): + # Update package name and version to reflect the KB name/ver + print(f" KB match for {package.name} {package.version}") + kb_matches += 1 + matchname = kb_match['componentName'] + matchver = kb_match['versionName'] + else: + print(f" No KB match for {package.name} {package.version}") + else: + # No external references field was provided + nopurl += 1 + print(f" No pURL provided for {package.name} {package.version}") + + # find_comp_import_events checks the imported name-version + bom_comp = find_comp_import_events(match_dict, package.name, package.version) + if bom_comp: + # bom_comp is the matched comp/ver string + bom_packages[bom_comp] = bom_packages.get(bom_comp, 0) + 1 + packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 + bom_matches += 1 + print(f" Found component in bom import-events: {matchname} {matchver}") + continue + else: + # Next look for the matchname-matchver in the BOM + # component search. The component name-version may have been + # updated above to reflect the pURL or KB matched name. + bom_comp = find_comp_in_bom(matchname, matchver, version) + if bom_comp: + bom_packages[bom_comp] = bom_packages.get(bom_comp, 0) + 1 + packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 + bom_matches += 1 + print(f" Found component in BOM: {matchname} {matchver}") + continue + + # If we've gotten this far, the package is not in the BOM. + # Now we need to figure out: + # - Is it already in the KB and we need to add it? + # - Do we need to add a custom component? + # - Do we need to add a version to an existing custom component? + not_in_bom += 1 + print(f" Not present in BOM: {matchname} {matchver}") + packages[matchname+matchver] = packages.get(matchname+matchver, 0) + 1 + + # Missing component data to write to a file for reference + comp_data = { + "name": package.name, + "spdx_id": package.spdx_id, + "version": package.version, + "origin": extref + } + comps_out.append(comp_data) + + # KB match was successful, but it wasn't in the BOM for some reason + if kb_match: + kb_match_added_to_bom += 1 + print(f" WARNING: {matchname} {matchver} found in KB but not in SBOM - adding it") + # kb_match['version'] contains the component url to add + add_to_sbom(proj_version_url, kb_match['version']) + # short-circuit the rest + continue + + # Check if custom component already exists + comp_url, comp_ver_url = find_cust_comp(package.name, package.version) + + if not comp_url: + # Custom component did not exist, so create it + cust_comp_count += 1 + comp_ver_url = create_cust_comp(package.name, package.version, + license_name) + elif comp_url and not comp_ver_url: + # Custom component existed, but not the version we care about + cust_ver_count += 1 + print(f" Adding version {package.version} to custom component {package.name}") + comp_ver_url = create_cust_comp_ver(comp_url, package.version, \ + license_name) + else: + print(" Custom component already exists, not in SBOM") + + # Shouldn't be possible + assert(comp_ver_url), f"No component URL found for {package.name} {package.version}" + + print(f" Adding component to SBOM: {package.name} aka {matchname} {package.version}") + cust_added_to_bom += 1 + add_to_sbom(proj_version_url, comp_ver_url) + + # Save unmatched components + if outfile: + json.dump(comps_out, outfile) + outfile.close() + + print("\nStats: ") + print("------") + print(f" SPDX packages processed: {package_count}") + # package_count above could have repeated packages in it + print(f" Unique packages processed: {len(packages)}") + print(f" Skipped: {skip_count}") + print(f" Packages missing purl or KBID: {nopurl}") + print(f" BOM matches: {bom_matches}") + print(f" Unique BOM matches: {len(bom_packages)}") + print(f" KB matches: {kb_matches}") + print(f" Custom components created: {cust_comp_count}") + print(f" Custom component versions created: {cust_ver_count}") + print(f" Packages missing from BOM: {not_in_bom}") + print(f" Custom components added to BOM: {cust_added_to_bom}") + print(f" KB matches added to BOM: {kb_match_added_to_bom}") + #pprint(packages) + #pprint(bom_packages) + +if __name__ == "__main__": + sys.exit(spdx_main_parse_args()) diff --git a/examples/client/recursive_delete_project.py b/examples/client/recursive_delete_project.py new file mode 100644 index 00000000..921d9cf9 --- /dev/null +++ b/examples/client/recursive_delete_project.py @@ -0,0 +1,168 @@ +''' +Created: Nov 23, 2023 +Author: mkumykov + +Copyright (c) 2023 - Synopsys, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This script will remove project hierarchies from a Black Duck server. + +usage: python3 recursive_delete_project.py [-h] -u BASE_URL -t TOKEN_FILE [-nv] [-p PROJECT_NAME] [-pv VERSION_NAME] [-pl PROJECT_LIST_FILE] + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -nv, --no-verify Disable TLS certificate verification + -p PROJECT_NAME, --project-name PROJECT_NAME + Project Name + -pv VERSION_NAME, --version-name VERSION_NAME + Version Name + -pl PROJECT_LIST_FILE, --project-list-file PROJECT_LIST_FILE + File containing project name list + +Options -pl and -p can not be used at the same time + +Examples: + +remove project with all sub-projects + + python3 recursive_delete_project.py -u BASE_URL -t TOKEN_FILE -nv -p PROJECT_NAME + +remove project version with all sub-projects, keep sub-projects still in use intact + + python3 recursive_delete_project.py -u BASE_URL -t TOKEN_FILE -nv -p PROJECT_NAME -pv VERSION_NAME + +remove projects listed in a file + + python3 recursive_delete_project.py -u BASE_URL -t TOKEN_FILE -nv -pl PROJECT_LIST_FILE + +''' + +import argparse +import json +import logging +import sys +import arrow + +from blackduck import Client +from pprint import pprint,pformat + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.INFO) +logging.getLogger("urllib3").setLevel(logging.INFO) +logging.getLogger("blackduck").setLevel(logging.INFO) + +strict = False + +def remove_project_structure(project_name): + project = find_project_by_name(project_name) + if not project: + logging.info(f"Project {project_name} does not exist.") + return + versions = bd.get_resource('versions', project) + for version in versions: + remove_project_version_structure(project_name, version['versionName']) + +def remove_project_version_structure(project_name, version_name): + project = find_project_by_name(project_name) + if not project: + logging.info(f"Project {project_name} does not exist.") + return + num_versions = bd.get_resource('versions', project, items=False)['totalCount'] + version = find_project_version_by_name(project,version_name) + if not version: + logging.info(f"Project {project_name} with version {version_name} does not exist.") + return + components = [ + c for c in bd.get_resource('components',version) if c['componentType'] == "SUB_PROJECT" + ] + logging.info(f"Project {project_name}:{version_name} has {len(components)} subprojects") + for component in components: + component_name = component['componentName'] + component_version_name = component['componentVersionName'] + logging.info(f"Removing subproject {component_name} from {project_name}:{version_name}") + component_url = component['_meta']['href'] + response = bd.session.delete(component_url) + logging.info(f"Operation completed with {response}") + remove_project_version_structure(component_name, component_version_name) + logging.info(f"Removing {project_name}:{version_name}") + if num_versions > 1: + response = bd.session.delete(version['_meta']['href']) + else: + response = bd.session.delete(project['_meta']['href']) + logging.info(f"Operation completed with {response}") + +def find_project_by_name(project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + + + +def parse_command_args(): + + parser = argparse.ArgumentParser("python3 recursive_delete_project.py") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + group = parser.add_mutually_exclusive_group() + group.add_argument("-p", "--project-name", required=False, help="Project Name") + parser.add_argument("-pv", "--version-name", required=False, help="Version Name") + group.add_argument("-pl", "--project-list-file", required=False, help="File containing project name list") + return parser.parse_args() + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + global bd + global scan_params + scan_params = [] + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + logging.info(f"{args}") + + if not (args.project_list_file or args.project_name): + logging.error("Project Name or a File containing Project Names should be specified") + return + + if args.project_list_file: + with open(args.project_list_file) as file: + lines = [line.rstrip() for line in file] + for line in lines: + remove_project_structure(line) + elif args.version_name: + remove_project_version_structure(args.project_name, args.version_name) + else: + remove_project_structure(args.project_name) + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/examples/client/refresh_project_copyrights.py b/examples/client/refresh_project_copyrights.py new file mode 100644 index 00000000..8b251b77 --- /dev/null +++ b/examples/client/refresh_project_copyrights.py @@ -0,0 +1,392 @@ +# Iterate through components within a named project (or all) and named version (or all) +# and refresh the copyrights of each - equivalent to clicking the UI refresh button +# Use --debug for some feedback on progress +# Ian Ashworth, May 2025 +# +import http.client +import signal +from sys import api_version +import sys +import csv +import datetime +from blackduck import Client +import argparse +import logging +from pprint import pprint +import array as arr + +from urllib3.exceptions import ReadTimeoutError + +http.client._MAXHEADERS = 1000 + +job_status = 0 + +logging.basicConfig( + level=logging.INFO, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +# initialise +all_my_comp_data = [] +my_statistics = {} + + +def RepDebug(level, msg): + if hasattr(args, 'debug') and level <= args.debug: + print("dbg{" + str(level) + "} " + msg) + return True + return False + +def RepWarning(msg): + print("WARNING: " + msg) + return True + +def CompleteTask(job_status): + now = datetime.datetime.now() + my_statistics['_jobStatus'] = job_status + + print('Finished: %s' % now.strftime("%Y-%m-%d %H:%M:%S")) + print('Summary:') + pprint(my_statistics) + + # if dumping data + if args.dump_data: + # if outputting to a CSV file + if args.csv_file: + '''Note: See the BD API doc and in particular .../api-doc/public.html#_bom_vulnerability_endpoints + for a complete list of the fields available. The below code shows a subset of them just to + illustrate how to write out the data into a CSV format. + ''' + logging.info(f"Exporting {len(all_my_comp_data)} records to CSV file {args.csv_file}") + + with open(args.csv_file, 'w') as csv_f: + field_names = [ + 'Component', + 'Component Version', + 'Status', + 'Url' + ] + + writer = csv.DictWriter(csv_f, fieldnames=field_names) + writer.writeheader() + + for my_comp_data in all_my_comp_data: + row_data = { + 'Component': my_comp_data['componentName'], + 'Component Version': my_comp_data['componentVersion'], + 'Status': my_comp_data['status'], + 'Url': my_comp_data['url'] + } + writer.writerow(row_data) + else: + # print to screen + pprint(all_my_comp_data) + +def SignalHandler(sig, frame): + # Complete the work + print("Ctrl+C detected!") + + # tidy up and complete the job + CompleteTask(1) + sys.exit(job_status) + +# ------------------------------------------------------------------------------ +# register the signal handler +signal.signal(signal.SIGINT, SignalHandler) + + +# Parse command line arguments +parser = argparse.ArgumentParser("Refresh copyrights for project/version components") + +parser.add_argument("--base-url", required=True, help="BD Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("--token-file", dest='token_file', required=True, help="File containing your access token") + +parser.add_argument("--dump-data", dest='dump_data', action='store_true', help="Retain analysed data") +parser.add_argument("--csv-file", dest='csv_file', help="File name for dumped data formatted as CSV") + +parser.add_argument("--project", dest='project_name', help="Project name") +parser.add_argument("--version", dest='version_name', help="Version name") + +parser.add_argument("--max-projects", dest='max_projects', type=int, help="Maximum number of projects to inspect else all") +parser.add_argument("--max-versions-per-project", dest='max_versions_per_project', type=int, help="Maximum versions per project to inspect else all") +parser.add_argument("--max-components", dest='max_components', type=int, help="Maximum components to inspect in total else all") + +parser.add_argument("--skip-projects", dest='skip_projects', type=int, help="Skip first 'n' projects to inspect") + +parser.add_argument("--debug", dest='debug', type=int, default=0, help="Debug verbosity (0=none 'n'=level)") +parser.add_argument("--dryrun", dest='dry_run', type=int, default=0, help="Dry run test (0=no 1=yes)") + +parser.add_argument("--no-verify", dest='verify', action='store_false', help="Disable TLS certificate verification") +parser.add_argument("--timeout", default=60, type=int, help="Adjust the (HTTP) session timeout value (default: 60s)") +parser.add_argument("--retries", default=3, type=int, help="Adjust the number of retries on failure (default: 3)") + +args = parser.parse_args() + +# open the access token file +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +# access the Black Duck platform +bd = Client( + base_url=args.base_url, + verify=args.verify, + token=access_token, + timeout=args.timeout, + retries=args.retries, +) + + +str_unknown = "n/a" + +str_unknown = "n/a" + +# version of components API to call +comp_api_version = 6 + +comp_accept_version = "application/vnd.blackducksoftware.bill-of-materials-" + str(comp_api_version) + "+json" +#comp_accept_version = "application/json" + +comp_content_type = comp_accept_version + +# header keys +comp_lc_keys = {} +comp_lc_keys['accept'] = comp_accept_version +comp_lc_keys['content-type'] = comp_accept_version + +# keyword arguments to pass to API call +comp_kwargs={} +comp_kwargs['headers'] = comp_lc_keys + + +# version of API to call +refresh_api_version = 4 + +refresh_accept_version = "application/vnd.blackducksoftware.copyright-" + str(refresh_api_version) + "+json" +#refresh_accept_version = "application/json" + +refresh_content_type = refresh_accept_version + + +# header keys +refresh_lc_keys = {} +refresh_lc_keys['accept'] = refresh_accept_version +refresh_lc_keys['content-type'] = refresh_accept_version + +# keyword arguments to pass to API call +refresh_kwargs={} +refresh_kwargs['headers'] = refresh_lc_keys + + +# zero our main counters +my_statistics['_cntProjects'] = 0 +my_statistics['_cntVersions'] = 0 +my_statistics['_cntComponents'] = 0 +my_statistics['_cntOrigins'] = 0 + +my_statistics['_cntRefresh'] = 0 +my_statistics['_cntNoOrigins'] = 0 +my_statistics['_cntNoIDs'] = 0 +my_statistics['_cntSkippedProjects'] = 0 +my_statistics['_jobStatus'] = 0 + +# record any control values +if args.project_name: + my_statistics['_namedProject'] = args.project_name +if args.version_name: + my_statistics['_namedVersion'] = args.version_name + +if args.max_projects: + my_statistics['_maxProjects'] = args.max_projects +if args.max_versions_per_project: + my_statistics['_maxVersionsPerProject'] = args.max_versions_per_project +if args.max_components: + my_statistics['_maxComponents'] = args.max_components + +now = datetime.datetime.now() +print('Started: %s' % now.strftime("%Y-%m-%d %H:%M:%S")) + +# check named project of specific interest +if args.project_name: + params = { + 'q': [f"name:{args.project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] + + # must exist + assert len(projects) > 0, f"There should be at least one - {len(projects)} project(s) noted" +else: + # all projects are in scope + projects = bd.get_resource('projects') + + +cnt_project = 0 +cnt_call = 0 + +# loop through projects list +for this_project in projects: + + cnt_project += 1 + + # check if we are skipping over this project + if args.skip_projects and cnt_project <= args.skip_projects: + my_statistics['_cntSkippedProjects'] += 1 + RepDebug(1, 'Skipping project [%d] [%s]' % (cnt_project, this_project['name'])) + continue + + # check if we have hit any limit + if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + RepDebug(1, 'Reached component limit [%d]' % args.max_components) + break + + if args.max_projects and my_statistics['_cntProjects'] >= args.max_projects: + RepDebug(1, 'Reached project limit [%d]' % args.max_projects) + break + + # process this project + my_statistics['_cntProjects'] += 1 + RepDebug(1, '## Project: [%d] [%s]' % (cnt_project, this_project['name'])) + + if args.version_name: + # note the specific project version of interest + params = { + 'q': [f"versionName:{args.version_name}"] + } + versions = [v for v in bd.get_resource('versions', this_project, params=params) if v['versionName'] == args.version_name] + + # it must exist + assert len(versions) > 0, f"There should be at least one - {len(versions)} version(s) noted" + else: + # all versions for this project are in scope + versions = bd.get_resource('versions', this_project) + + nVersionsPerProject = 0 + + for this_version in versions: + + # check if we have hit any limit + if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + RepDebug(1, 'Reached component limit [%d]' % args.max_components) + break + + if args.max_versions_per_project and nVersionsPerProject >= args.max_versions_per_project: + RepDebug(1, 'Reached versions per project limit [%d]' % args.max_versions_per_project) + break + + nVersionsPerProject += 1 + my_statistics['_cntVersions'] += 1 + + # Announce +# logging.debug(f"Found {this_project['name']}:{this_version['versionName']}") + RepDebug(3, ' Version: [%s]' % this_version['versionName']) + + + # iterate through all components for this project version + for this_comp_data in bd.get_resource('components', this_version, **comp_kwargs): + + if args.max_components and my_statistics['_cntComponents'] >= args.max_components: + break + + my_statistics['_cntComponents'] += 1 + + if this_comp_data.get("componentName"): + comp_name = this_comp_data['componentName'] + else: + comp_name = str_unknown + + if this_comp_data.get("componentVersionName"): + comp_version_name = this_comp_data['componentVersionName'] + else: + comp_version_name = str_unknown + + comp_label = "{} ({})".format(comp_name, comp_version_name) + + RepDebug(4, ' Component: [%s]' % comp_label) + + if this_comp_data['inputExternalIds'].__len__() > 0: + inputExternalIds = this_comp_data['inputExternalIds'][0] + else: + my_statistics['_cntNoIDs'] += 1 + inputExternalIds = str_unknown + RepDebug(2, ' ID: [%s]' % inputExternalIds) + + + # refresh the copyrights for this component-origin + if this_comp_data['origins'].__len__() > 0: + + n_origin = 0 + + for this_origin in this_comp_data['origins']: + + n_origin += 1 + my_statistics['_cntOrigins'] += 1 + + if this_origin.get('externalId'): + origin_id = this_origin['externalId'] + else: + origin_id = str_unknown + + url = this_origin['origin'] + + # refresh with end point + url += "/copyrights-refresh" + + status = -1 + cnt_call += 1 + call_id = "{}.{}".format(cnt_project, cnt_call) + + if args.dry_run != 0: + RepDebug(2, ' DryRun: %s - origin - no [%d] id [%s] url [%s]' % (call_id, n_origin, origin_id, url)) + else: + RepDebug(3, + ' Origin: %s - origin - no [%d] id [%s] url [%s]' % (call_id, n_origin, origin_id, url)) + try: + response = bd.session.put(url, data=None, **refresh_kwargs) + RepDebug(5,'Refresh response: origin [%s] [%s]' % (this_origin, response)) + my_statistics['_cntRefresh'] += 1 + status= 0 + + except Exception: + print('Failed to confirm copyrights refresh') + status = 1 + + + # if recording the data - perhaps outputting to a CSV file + if args.dump_data: + my_data = {} + my_data['componentName'] = this_comp_data['componentName'] + my_data['componentVersion'] = this_comp_data['componentVersionName'] + my_data['status'] = status + my_data['url'] = url + + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + + # add to our list + all_my_comp_data.append(my_data) + + else: + # no origins defined + RepWarning('No origin(s) defined for [%s]' % comp_label) + my_statistics['_cntNoOrigins'] += 1 + origin_id = '' + status = 3 + url = 'n/a' + + # if recording the data + if args.dump_data: + my_data = {} + my_data['componentName'] = comp_name + my_data['componentVersion'] = comp_version_name + my_data['status'] = status + my_data['url'] = url + + if hasattr(args, 'debug') and 5 <= args.debug: + pprint(my_data) + + # add to our list + all_my_comp_data.append(my_data) + +# end of processing loop + +CompleteTask(0) +#end diff --git a/examples/client/remap_codelocations.py b/examples/client/remap_codelocations.py new file mode 100644 index 00000000..089a97bd --- /dev/null +++ b/examples/client/remap_codelocations.py @@ -0,0 +1,91 @@ +""" +Created on july 11, 2024 + +@author: Dinesh Ravi + +Remap codelocations from a project version to another project version + +""" + +from blackduck import Client + +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s", +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument( + "-u", "--bd_url", help="Hub server URL e.g. https://your.blackduck.url" +) +parser.add_argument( + "-t", "--token-file", help="File name of a file containing access token" +) +parser.add_argument( + "-nv", + "--no-verify", + dest="verify", + action="store_false", + help="disable TLS certificate verification", +) +parser.add_argument("project_name") +parser.add_argument("version_name") +parser.add_argument("update_pv_url") + + +args = parser.parse_args() + +logging.basicConfig( + format="%(asctime)s:%(levelname)s:%(message)s", + stream=sys.stderr, + level=logging.DEBUG, +) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, "r") as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) + +params = {"q": [f"name:{args.project_name}"]} +projects = [ + p + for p in bd.get_resource("projects", params=params) + if p["name"] == args.project_name +] +assert ( + len(projects) == 1 +), f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +params = {"q": [f"versionName:{args.version_name}"]} +versions = [ + v + for v in bd.get_resource("versions", project, params=params) + if v["versionName"] == args.version_name +] +assert ( + len(versions) == 1 +), f"There should be one, and only one version named {args.version_name}. We found {len(versions)}" +version = versions[0] + +logging.debug(f"Found {project['name']}:{version['versionName']}") + + +codelocations = bd.get_resource("codelocations", version) +# logging.info(f"Total Code locations '{len(list(codelocations))}'") +for codelocation in codelocations: + logging.debug(f"Un-mapping code location {codelocation['name']}") + url = codelocation["_meta"]["href"] + codelocation["mappedProjectVersion"] = args.update_pv_url + result = bd.session.put(url, json=codelocation) + logging.info(f"Code location '{codelocation['name']}' unmap status {result}") diff --git a/examples/client/sbomify.py b/examples/client/sbomify.py new file mode 100644 index 00000000..ffdd2deb --- /dev/null +++ b/examples/client/sbomify.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +''' +Created: Apr 2, 2024 +Author: @kumykov + +Copyright (c) 2024, Synopsys, Inc. +http://www.synopsys.com/ + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + +usage: sbomify.py [-h] -u BASE_URL -t TOKEN_FILE [-nv] -sp SOURCE_PROJECT -sv SOURCE_VERSION -tp TARGET_PROJECT -tv TARGET_VERSION + [-tg TARGET_PROJECT_GROUP] [-c] [--sbom-type SBOM_TYPE] + +Generate and download SBOM for the source project version +and upload it to the target project version + +options: + -h, --help show this help message and exit + -u BASE_URL, --base-url BASE_URL + Hub server URL e.g. https://your.blackduck.url + -t TOKEN_FILE, --token-file TOKEN_FILE + File containing access token + -nv, --no-verify Disable TLS certificate verification + -sp SOURCE_PROJECT, --source-project SOURCE_PROJECT + Source Project Name + -sv SOURCE_VERSION, --source-version SOURCE_VERSION + Source Project Version Name + -tp TARGET_PROJECT, --target-project TARGET_PROJECT + Target Project Name + -tv TARGET_VERSION, --target-version TARGET_VERSION + Target Project Version Name + -tg TARGET_PROJECT_GROUP, --target-project-group TARGET_PROJECT_GROUP + Project Group to use for target + -c, --create-target Create target project version if does not exist + --sbom-type {SPDX_22,SPDX_23,CYCLONEDX_13,CYCLONEDX_14} + SBOM type to use for transaction + +Black Duck examples collection + + +''' +import argparse +import io +import json +import sys +import logging +import time + +from zipfile import ZipFile +from blackduck import Client + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + + +def find_project_by_name(bd, project_name): + params = { + 'q': [f"name:{project_name}"] + } + projects = [p for p in bd.get_resource('projects', params=params) if p['name'].casefold() == project_name.casefold()] + if len(projects) == 1: + return projects[0] + else: + return None + +def find_project_version_by_name(bd, project, version_name): + params = { + 'q': [f"versionName:{version_name}"] + } + versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name] + if len(versions) == 1: + return versions[0] + else: + return None + +def find_or_create_project_group(bd, group_name): + url = '/api/project-groups' + params = { + 'q': [f"name:{group_name}"] + } + groups = [p for p in bd.get_items(url, params=params) if p['name'] == group_name] + if len(groups) == 0: + headers = { + 'Accept': 'application/vnd.blackducksoftware.project-detail-5+json', + 'Content-Type': 'application/vnd.blackducksoftware.project-detail-5+json' + } + data = { + 'name': group_name + } + response = bd.session.post(url, headers=headers, json=data) + return response.headers['Location'] + else: + return groups[0]['_meta']['href'] + +def create_project_version(bd, project_name,version_name,project_group, nickname = None): + version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name} + if nickname: + version_data['nickname'] = nickname + url = '/api/projects' + project = find_project_by_name(bd, project_name) + if project: + data = version_data + url = project['_meta']['href'] + '/versions' + else: + data = {"name": project_name, + "projectGroup": find_or_create_project_group(bd, project_group), + "versionRequest": version_data} + return bd.session.post(url, json=data) + +def locate_project_version(bd, project_name, version_name, group="Black Duck Project Groups", create=False): + project = find_project_by_name(bd, project_name) + version = None + if project: + version = find_project_version_by_name(bd, project, version_name) + if version: + pass + elif create: + version = create_project_version(bd, project_name, version_name, group) + else: + pass + elif create: + response = create_project_version(bd, project_name, version_name, group) + logging.info(f"Project {project_name} : {version_name} creation completed with {response}") + if response.ok: + project = find_project_by_name(bd, project_name) + version = find_project_version_by_name(bd, project, version_name) + return version + +def create_sbom_report(bd, version, type, include_subprojects): + post_data = { + 'reportFormat': "JSON", + 'sbomType': type, + 'includeSubprojects': include_subprojects + } + sbom_reports_url = version['_meta']['href'] + "/sbom-reports" + + bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json" + r = bd.session.post(sbom_reports_url, json=post_data) + if (r.status_code == 403): + logging.debug("Authorization Error - Please ensure the token you are using has write permissions!") + r.raise_for_status() + location = r.headers.get('Location') + assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from" + return location + +def download_report(bd, location, retries): + report_id = location.split("/")[-1] + if retries: + logging.debug(f"Retrieving generated report from {location}") + response = bd.session.get(location) + report_status = response.json().get('status', 'Not Ready') + if response.status_code == 200 and report_status == 'COMPLETED': + response = bd.session.get(location + "/download.zip", headers={'Content-Type': 'application/zip', 'Accept':'application/zip'}) + if response.status_code == 200: + return response.content + else: + logging.error("Ruh-roh, not sure what happened here") + return None + else: + logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {retries} seconds then retrying...") + time.sleep(60) + retries -= 1 + return download_report(bd, location, retries) + else: + logging.debug(f"Failed to retrieve report {report_id} after multiple retries") + return None + +def produce_online_sbom_report(bd, project_name, project_version_name, sbom_type): + project = find_project_by_name(bd, project_name) + logging.debug(f"Project {project['name']} located") + version = find_project_version_by_name(bd, project, project_version_name) + logging.debug(f"Version {version['versionName']} located") + location = create_sbom_report(bd, version, sbom_type, True) + logging.debug(f"Created SBOM report of type {sbom_type} for project {project_name}, version {project_version_name} at location {location}") + sbom_data_zip = download_report(bd, location, 60) + logging.debug(f"Deleting report from Black Duck {bd.session.delete(location)}") + zip=ZipFile(io.BytesIO(sbom_data_zip), "r") + sbom_data = {name: zip.read(name) for name in zip.namelist()} + filename = [i for i in sbom_data.keys() if i.endswith(".json")][0] + return json.loads(sbom_data[filename]) + +def upload_sbom_file(bd, project_name, version_name, sbom_data): + if sbom_data.get('bomFormat', None) == "CycloneDX": + mime_type = 'application/vnd.cyclonedx' + elif sbom_data.get('spdxVersion', None): + mime_type = 'application/spdx' + else: + mime_type = None + if not mime_type: + logging.error(f"Could not identify file content for SBOM") + sys.exit(1) + logging.info(f"Mime type {mime_type} will be used for SBOM upload") + files = {"file": ('sbom.json', json.dumps(sbom_data).encode('utf-8'), mime_type)} + fields = {"projectName": project_name, "versionName": version_name} + response = bd.session.post("/api/scan/data", files = files, data=fields) + logging.info(f"SBOM Upload completed with {response}") + if response.status_code == 409: + logging.info(f"File SBOM is already mapped to a different project version") + +def sbomify(bd, args): + source = locate_project_version(bd, args.source_project, args.source_version) + if not source: + logging.error(f"Source project {args.source_project} : {args.source_version} not found. Exiting.") + sys.exit(1) + logging.info(f"Located source project {args.source_project} : {args.source_version}") + sbom = produce_online_sbom_report(bd, args.source_project, args.source_version, args.sbom_type) + bd.session.headers.pop('Content-Type') + target = locate_project_version(bd, args.target_project, args.target_version, group=args.target_project_group, create=args.create_target) + if not target: + logging.error(f"Target project {args.target_project} : {args.target_version} not found. Exiting.") + sys.exit(1) + logging.info(f"Located target project {args.target_project} : {args.target_version}") + upload_sbom_file(bd, args.target_project, args.target_version, sbom) + +def parse_command_args(): + parser = argparse.ArgumentParser(prog = "sbomify.py", description="Generate and download SBOM and upload to the target project version", epilog="Blackduck examples collection") + parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url") + parser.add_argument("-t", "--token-file", required=True, help="File containing access token") + parser.add_argument("-nv", "--no-verify", action='store_false', help="Disable TLS certificate verification") + parser.add_argument("-sp", "--source-project", required=True, help="Source Project Name") + parser.add_argument("-sv", "--source-version", required=True, help="Source Project Version Name") + parser.add_argument("-tp", "--target-project", required=True, help="Target Project Name") + parser.add_argument("-tv", "--target-version", required=True, help="Target Project Version Name") + parser.add_argument("-tg", "--target-project-group", required=False, default='Black Duck Project Groups', help="Project Group to use for target") + parser.add_argument("-c", "--create-target", action='store_true', help="Create target project version if does not exist") + parser.add_argument("--sbom-type", required=False, default='SPDX_23', choices=["SPDX_22", "SPDX_23", "CYCLONEDX_13", "CYCLONEDX_14"], help="SBOM type to use for transaction") + + return parser.parse_args() + +def main(): + args = parse_command_args() + with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + bd = Client(base_url=args.base_url, token=access_token, verify=args.no_verify, timeout=60.0, retries=4) + sbomify(bd, args) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/client/unmap_codelocations.py b/examples/client/unmap_codelocations.py new file mode 100644 index 00000000..a3d080bd --- /dev/null +++ b/examples/client/unmap_codelocations.py @@ -0,0 +1,66 @@ +''' +Created on Jan 18, 2024 + +@author: kumykov + +Unmap codelocations from a project version + +''' + +from blackduck import Client + +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument("-u", "--bd-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", help="File name of a file containing access token") +parser.add_argument("-nv", '--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("project_name") +parser.add_argument("version_name") + +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) + +params = { + 'q': [f"name:{args.project_name}"] +} +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +params = { + 'q': [f"versionName:{args.version_name}"] +} +versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name] +assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}" +version = versions[0] + +logging.debug(f"Found {project['name']}:{version['versionName']}") + +codelocations = bd.get_resource('codelocations', version) + +for codelocation in codelocations: + logging.debug(f"Un-mapping code location {codelocation['name']}") + url = codelocation['_meta']['href'] + codelocation['mappedProjectVersion'] = None + result = bd.session.put(url, json=codelocation) + logging.info(f"Code location '{codelocation['name']}' unmap status {result}") \ No newline at end of file diff --git a/examples/client/update_component_version.py b/examples/client/update_component_version.py new file mode 100644 index 00000000..df56cf46 --- /dev/null +++ b/examples/client/update_component_version.py @@ -0,0 +1,205 @@ +''' +Created on Jan 22, 2024 + +@author: pedapati + +Update component version info for BOM Components with Unknown Versions based on matched filename in a given project version + +''' + +from blackduck import Client + +import requests +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +import urllib3 +import urllib.parse + +NAME = 'update_component_version.py' +VERSION = '2024-01-22' + +print(f'{NAME} ({VERSION}). Copyright (c) 2023 Synopsys, Inc.') + + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument("-u", "--bd-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", help="File name of a file containing access token") +parser.add_argument("-nv", '--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("project_name") +parser.add_argument("version_name") + +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) + +params = { + 'q': [f"name:{args.project_name}"] +} +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] +project_id = project["_meta"]["href"].split("/")[-1] +print("Project ID: " + project_id) + +params = { + 'q': [f"versionName:{args.version_name}"] +} +versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name] +assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}" +version = versions[0] +version_id = version["_meta"]["href"].split("/")[-1] +print("Version ID: " + version_id) + +logging.debug(f"Found {project['name']}:{version['versionName']}") + +def update_bom_unknown_versions(bd, project_id, version_id): + limit = 1000 + offset = 0 + paginated_url = f"{bd.base_url}/api/projects/{project_id}/versions/{version_id}/components?limit={limit}&offset={offset}&filter=unknownVersion:true" + print("Looking for BOM Components with Unknown Versions: " + paginated_url) + print() + components_json = bd.session.get(paginated_url).json() + total = str(components_json["totalCount"]) + print("Found " + total + " components with unknown versions") + print() + for component in components_json["items"]: + comp_name =component["componentName"] + print("Processing Component: " + comp_name) + comp_url = component["component"] + comp_bom_url = component["_meta"]["href"] + matched_files_url = component["_meta"]["href"] + "/matched-files" + matched_file_json = bd.session.get(matched_files_url).json() + archivecontext = matched_file_json["items"][0]["filePath"]["archiveContext"] + filename = matched_file_json["items"][0]["filePath"]["fileName"] + ## Extract Component Name and Version from archivecontext to do a KB lookup + archive_strip = archivecontext.strip("/,!") + archive_partition = archive_strip.rpartition("-") + archive_final_list = archive_partition[0].rpartition("-") + kb_file_lookup_name = archive_final_list[0] + kb_comp_lookup_version = archive_final_list[2] + print("Processing Component Version: " + kb_comp_lookup_version) + ## KB Lookup via Component Name + components_url = bd.base_url + "/api/components/autocomplete" + query = { "q": comp_name, + "filter": "componentType:kb_component" + } + url = f"{components_url}?{urllib.parse.urlencode(query)}" + headers = {'Accept': '*/*'} + name_match = bd.session.get(url, headers=headers).json() + # Filtering results for exact name match + exact_name_match = [x for x in name_match['items'] if x['name']==comp_name] + if len(exact_name_match) == 0 : + logging.debug(f"Component {comp_name} is not found in the KB") + return + else: + logging.debug(f"Component {comp_name} is found in the KB") + if kb_comp_lookup_version: + first_match_successful = False + # second_match_successful = False + for match in exact_name_match: # handling OSS components that share same name + url = match['_meta']['href']+'/versions?q=versionName:' + kb_comp_lookup_version + headers = {'Accept': 'application/vnd.blackducksoftware.summary-1+json'} + # Producing version matches + version_match = bd.session.get(url, headers=headers).json() + if version_match['totalCount'] > 0: + print(version_match["items"][0]["versionName"]) + print("Found version: " + kb_comp_lookup_version + " in the KB for component " + comp_name) + print("Updating component version for " + comp_name + " to " + kb_comp_lookup_version ) + # component_url = version_match[] + # print(version_match) + component_version_url = version_match['items'][0]['_meta']['href'] + component_url = component_version_url[:component_version_url.index("versions")-1] + # print(component_url) + post_data = {"component": component_url, "componentVersion": component_version_url} + headers = {'Accept': 'application/vnd.blackducksoftware.bill-of-materials-6+json', 'Content-Type': 'application/vnd.blackducksoftware.bill-of-materials-6+json'} + response = bd.session.put(comp_bom_url, headers=headers, data=json.dumps(post_data)) + # print(response) + if response.status_code == 200: + message = f"{response}" + print("Successfully updated " + comp_name + " with version " + kb_comp_lookup_version) + else: + message = f"{response.json()}" + logging.debug(f"Updating BOM component {comp_name} {kb_comp_lookup_version} failed with: {message}") + first_match_successful = True + print("### Proceeding to next component") + print() + break + else: + print("No matching version " + kb_comp_lookup_version + " found for " + comp_name) + if not first_match_successful: + ## Trying to locate component name using source archive name + print("Proceeding to KB lookup via matched file name: " + kb_file_lookup_name) + components_url = bd.base_url + "/api/components/autocomplete" + query = { "q": kb_file_lookup_name, + "filter": "componentType:kb_component" + } + url = f"{components_url}?{urllib.parse.urlencode(query)}" + headers = {'Accept': '*/*'} + name_match = bd.session.get(url, headers=headers).json() + # Filtering results for exact name match + exact_name_match = [x for x in name_match['items'] if x['name']==kb_file_lookup_name] + if len(exact_name_match) == 0 : + logging.debug(f"File Match KB Component {kb_file_lookup_name} is not found in the KB") + print("### Proceeding to next component") + print() + continue + else: + logging.debug(f"File Match KB Component {kb_file_lookup_name} is found in the KB") + if kb_comp_lookup_version: + for match in exact_name_match: # handling OSS components that share same name + url = match['_meta']['href']+'/versions?q=versionName:' + kb_comp_lookup_version + headers = {'Accept': 'application/vnd.blackducksoftware.summary-1+json'} + # Producing version matches + version_match = bd.session.get(url, headers=headers).json() + if version_match['totalCount'] > 0: + print(version_match["items"][0]["versionName"]) + print("Found version: " + kb_comp_lookup_version + " in the KB for component " + kb_file_lookup_name) + print("Updating component version for " + kb_file_lookup_name + " to " + kb_comp_lookup_version ) + # component_url = version_match[] + # print(version_match) + component_version_url = version_match['items'][0]['_meta']['href'] + component_url = component_version_url[:component_version_url.index("versions")-1] + # print(component_url) + post_data = {"component": component_url, "componentVersion": component_version_url} + headers = {'Accept': 'application/vnd.blackducksoftware.bill-of-materials-6+json', 'Content-Type': 'application/vnd.blackducksoftware.bill-of-materials-6+json'} + response = bd.session.put(comp_bom_url, headers=headers, data=json.dumps(post_data)) + # print(response) + if response.status_code == 200: + message = f"{response}" + print("Successfully updated " + kb_file_lookup_name + " with version " + kb_comp_lookup_version) + # second_match_successful = True + else: + message = f"{response.json()}" + logging.debug(f"Updating BOM component {kb_file_lookup_name} {kb_comp_lookup_version} failed with: {message}") + print("### Proceeding to next component") + print() + break + else: + print("No matching version " + kb_comp_lookup_version + " found for " + kb_file_lookup_name) + print("### Proceeding to next component") + print() + + + +bom = update_bom_unknown_versions(bd, project_id, version_id) + + + diff --git a/examples/client/update_project_settings.py b/examples/client/update_project_settings.py new file mode 100644 index 00000000..b93a18e1 --- /dev/null +++ b/examples/client/update_project_settings.py @@ -0,0 +1,65 @@ +''' +Created on Jan 18, 2024 + +@author: kumykov + +Update project settings script. + +This script will modify project settings accessible via API + +Parameter list and their default values: + "customSignatureEnabled" : false, + "customSignatureDepth" : "5", + "unmatchedFileRetentionEnabled" : false, + +''' + +from blackduck import Client + +import argparse +import json +import logging +import sys +import time +from pprint import pprint + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +parser = argparse.ArgumentParser(sys.argv[0]) +parser.add_argument("-u", "--bd-url", help="Hub server URL e.g. https://your.blackduck.url") +parser.add_argument("-t", "--token-file", help="File name of a file containing access token") +parser.add_argument("-nv", '--no-verify', dest='verify', action='store_false', help="disable TLS certificate verification") +parser.add_argument("-cse", '--custom-signature-enabled', dest='cs_enable', action='store_true', help="enable custom signature flag") +parser.add_argument("-csd", '--custom-signature-depth', dest='cs_depth', required=False, default="5", help="set custom signature depth") +parser.add_argument("-ruf", '--retain-unmatched-files', dest='retain_uf', action='store_true', help="set retain unmatched files flag") +parser.add_argument("project_name") + +args = parser.parse_args() + +logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) +logging.getLogger("blackduck").setLevel(logging.WARNING) + +with open(args.token_file, 'r') as tf: + access_token = tf.readline().strip() + +bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify) +pprint (args.project_name) +params = { + 'q': [f"name:{args.project_name}"] +} +projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name] +assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}" +project = projects[0] + +url = project['_meta']['href'] +project['customSignatureEnabled'] = args.cs_enable +project['customSignatureDepth'] = args.cs_depth +project['unmatchedFileRetentionEnabled'] = args.retain_uf + +response = bd.session.put(url, json=project) +logging.info(f"Project setting update status {response}") diff --git a/examples/client/upload_bdio.py b/examples/client/upload_bdio.py index 8df294de..ff505032 100644 --- a/examples/client/upload_bdio.py +++ b/examples/client/upload_bdio.py @@ -65,6 +65,7 @@ def main(): files = {"file": open(args.filename,"rb")} response = bd.session.post("/api/scan/data", files = files) logging.info(response) + logging.info(response.headers) def parse_command_args(): parser = argparse.ArgumentParser(prog = "upload_bdio", description="Uploads BDIO file to a Blackduck server", epilog="Blackduck examples collection") diff --git a/examples/client/upload_sbom.py b/examples/client/upload_sbom.py index 931797da..fd726077 100644 --- a/examples/client/upload_sbom.py +++ b/examples/client/upload_sbom.py @@ -98,7 +98,7 @@ def find_or_create_project_group(group_name): else: return groups[0]['_meta']['href'] -def create_project_version(project_name,version_name,project_group, nickname = None): +def create_project_version(project_name, version_name, project_group, nickname = None): version_data = {"distribution": "EXTERNAL", "phase": "DEVELOPMENT", "versionName": version_name} if nickname: version_data['nickname'] = nickname @@ -120,7 +120,7 @@ def find_or_create_project_version(project_name, version_name, project_group): if version: pass else: - version = create_project_version(project_name, version_name) + version = create_project_version(project_name, version_name, project_group) else: version = create_project_version(project_name, version_name, project_group) project = find_project_by_name(project_name) @@ -130,22 +130,22 @@ def find_or_create_project_version(project_name, version_name, project_group): sys.exit(1) def get_sbom_mime_type(filename): + import json with open(filename, 'r') as f: - data = f.readlines() - content = " ".join(data) - if 'CycloneDX' in content: + data = json.load(f) + if data.get('bomFormat', None) == "CycloneDX": return 'application/vnd.cyclonedx' - if 'SPDX' in content: + elif data.get('spdxVersion', None): return 'application/spdx' return None def upload_sbom_file(filename, project, version, project_group): find_or_create_project_version(project, version, project_group) mime_type = get_sbom_mime_type(filename) - print (mime_type) if not mime_type: logging.error(f"Could not identify file content for {filename}") sys.exit(1) + logging.info(f"Mime type {mime_type} will be used for file {filename}") files = {"file": (filename, open(filename,"rb"), mime_type)} fields = {"projectName": project, "versionName": version} response = bd.session.post("/api/scan/data", files = files, data=fields) diff --git a/examples/create_api_token.py b/examples/create_api_token.py new file mode 100644 index 00000000..a0315ed7 --- /dev/null +++ b/examples/create_api_token.py @@ -0,0 +1,67 @@ +''' +Created on January 1, 2024 + +@author: dnichol + +Create an API token with either readonly or read and write access. + +To use this script. Firstly create a .restconfig.json file with either API token or basic auth (username/password) as per the examples : +https://github.com/blackducksoftware/hub-rest-api-python/blob/master/restconfig.json.example +https://github.com/blackducksoftware/hub-rest-api-python/blob/master/restconfig.json.api_token.example + +Then to run: +python create_api_token.py MyToken "My Token Description" + +It will output the token that is generated. If you would like the token to be read only add the -r flag to the command line. + +''' +import argparse +import json +import logging +from pprint import pprint +import sys + +from blackduck.HubRestApi import HubInstance + + +parser = argparse.ArgumentParser("Create an API token") +parser.add_argument("name") +parser.add_argument("description") +parser.add_argument("-r", "--readonly", action='store_true') + + +args = parser.parse_args() + +logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) +logging.getLogger("requests").setLevel(logging.WARNING) +logging.getLogger("urllib3").setLevel(logging.WARNING) + +hub = HubInstance() + +scope = ["read"] + +if not args.readonly : + scope = ["read", "write"] + +post_data = { + "name" : args.name, + "description" : args.description, + "scopes" : scope +} + +current_user = hub.get_current_user() +add_token_url = hub.get_link(current_user, "api-tokens") + +response = hub.execute_post(add_token_url, data=post_data) +if response.status_code == 201: + token_obj = response.json() + token=token_obj['token'] + logging.info("Added API token {} = {}".format(args.name, token)) +else: + logging.error("Failed to add API token {}, status code was {}".format( + args.name, response.status_code)) + + + + + diff --git a/examples/generate_csv_reports_for_project_version_enhanced.py b/examples/generate_csv_reports_for_project_version_enhanced.py new file mode 100644 index 00000000..86e18493 --- /dev/null +++ b/examples/generate_csv_reports_for_project_version_enhanced.py @@ -0,0 +1,246 @@ +''' +Created on Dec 19, 2018 +Updated on Sept 20, 2024 + +@author: gsnyder +@contributor: smiths + +Generate a CSV report for a given project-version and enhance with "File Paths", "How to Fix", and +"References and Related Links" +''' + +import argparse +import csv +import io +import json +import logging +import time +import zipfile +from blackduck.HubRestApi import HubInstance +from requests.exceptions import MissingSchema + +logging.basicConfig( + level=logging.DEBUG, + format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s" +) + +version_name_map = { + 'version': 'VERSION', + 'scans': 'CODE_LOCATIONS', + 'components': 'COMPONENTS', + 'vulnerabilities': 'SECURITY', + 'source': 'FILES', + 'cryptography': 'CRYPTO_ALGORITHMS', + 'license_terms': 'LICENSE_TERM_FULFILLMENT', + 'component_additional_fields': 'BOM_COMPONENT_CUSTOM_FIELDS', + 'project_version_additional_fields': 'PROJECT_VERSION_CUSTOM_FIELDS', + 'vulnerability_matches': 'VULNERABILITY_MATCH' +} + +all_reports = list(version_name_map.keys()) + +parser = argparse.ArgumentParser("A program to create reports for a given project-version") +parser.add_argument("project_name") +parser.add_argument("version_name") +parser.add_argument("-z", "--zip_file_name", default="reports.zip") +parser.add_argument("-r", "--reports", + default=",".join(all_reports), + help=f"Comma separated list (no spaces) of the reports to generate - {list(version_name_map.keys())}. Default is all reports.", + type=lambda s: s.upper()) +parser.add_argument('--format', default='CSV', choices=["CSV"], help="Report format - only CSV available for now") +parser.add_argument('-t', '--tries', default=5, type=int, help="How many times to retry downloading the report, i.e. wait for the report to be generated") +parser.add_argument('-s', '--sleep_time', default=30, type=int, help="The amount of time to sleep in-between (re-)tries to download the report") + +args = parser.parse_args() + +hub = HubInstance() + +class FailedReportDownload(Exception): + pass + +def download_report(location, filename, retries=args.tries): + report_id = location.split("/")[-1] + + for attempt in range(retries): + + # Wait for 30 seconds before attempting to download + print(f"Waiting 30 seconds before attempting to download...") + time.sleep(30) + + # Retries + print(f"Attempt {attempt + 1} of {retries} to retrieve report {report_id}") + + # Report Retrieval + print(f"Retrieving generated report from {location}") + response = hub.download_report(report_id) + + if response.status_code == 200: + with open(filename, "wb") as f: + f.write(response.content) + print(f"Successfully downloaded zip file to {filename} for report {report_id}") + return response.content + else: + print(f"Failed to retrieve report {report_id}") + if attempt < retries - 1: # If it's not the last attempt + wait_time = args.sleep_time + print(f"Waiting {wait_time} seconds before retrying...") + time.sleep(wait_time) + else: + print(f"Maximum retries reached. Unable to download report.") + + raise FailedReportDownload(f"Failed to retrieve report {report_id} after {retries} tries") + +def get_file_paths(hub, project_id, project_version_id, component_id, component_version_id, component_origin_id): + url = f"{hub.get_urlbase()}/api/projects/{project_id}/versions/{project_version_id}/components/{component_id}/versions/{component_version_id}/origins/{component_origin_id}/matched-files" + headers = { + "Accept": "application/vnd.blackducksoftware.bill-of-materials-6+json", + "Authorization": f"Bearer {hub.token}" + } + + logging.debug(f"Making API call to: {url}") + + try: + response = hub.execute_get(url) + if response.status_code == 200: + data = response.json() + file_paths = [] + for item in data.get('items', []): + file_path = item.get('filePath', {}) + composite_path = file_path.get('compositePathContext', '') + if composite_path: + file_paths.append(composite_path) + return file_paths + else: + logging.error(f"Failed to fetch matched files. Status code: {response.status_code}") + return [] + except Exception as e: + logging.error(f"Error making API request: {str(e)}") + return [] + +def get_vulnerability_details(hub, vulnerability_id): + url = f"{hub.get_urlbase()}/api/vulnerabilities/{vulnerability_id}" + + try: + response = hub.execute_get(url) + if response.status_code == 200: + data = response.json() + solution = data.get('solution', '') + references = [] + meta_data = data.get('_meta', {}) + links = meta_data.get('links', []) + for link in links: + references.append({ + 'rel': link.get('rel', ''), + 'href': link.get('href', '') + }) + return solution, references + else: + logging.error(f"Failed to fetch vulnerability details. Status code: {response.status_code}") + return '', [] + except Exception as e: + logging.error(f"Error making API request for vulnerability details: {str(e)}") + return '', [] + +def enhance_security_report(hub, zip_content, project_id, project_version_id): + logging.info(f"Enhancing security report for Project ID: {project_id}, Project Version ID: {project_version_id}") + + with zipfile.ZipFile(io.BytesIO(zip_content), 'r') as zin: + csv_files = [f for f in zin.namelist() if f.endswith('.csv')] + for csv_file in csv_files: + csv_content = zin.read(csv_file).decode('utf-8') + reader = csv.DictReader(io.StringIO(csv_content)) + + # Count total rows + total_rows = sum(1 for row in reader) + reader = csv.DictReader(io.StringIO(csv_content)) # Reset reader + + fieldnames = reader.fieldnames + ["File Paths", "How to Fix", "References and Related Links"] + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=fieldnames) + writer.writeheader() + + processed_components = 0 + skipped_components = 0 + + for index, row in enumerate(reader, 1): + print(f"\rProcessing row {index} of {total_rows} ({index/total_rows*100:.2f}%)", end='', flush=True) + + component_id = row.get('Component id', '') + component_version_id = row.get('Version id', '') + component_origin_id = row.get('Origin id', '') + vulnerability_id = row.get('Vulnerability id', '') + + if not all([component_id, component_version_id, component_origin_id]): + logging.warning(f"Missing component information. Component ID: {component_id}, Component Version ID: {component_version_id}, Origin ID: {component_origin_id}") + skipped_components += 1 + file_paths = [] + else: + file_paths = get_file_paths(hub, project_id, project_version_id, component_id, component_version_id, component_origin_id) + processed_components += 1 + + if vulnerability_id: + solution, references = get_vulnerability_details(hub, vulnerability_id) + else: + solution, references = '', [] + + row["File Paths"] = '; '.join(file_paths) if file_paths else "No file paths available" + row["How to Fix"] = solution + row["References and Related Links"] = json.dumps(references) + + writer.writerow(row) + + print("\nProcessing complete.") + + # Generate a unique filename for the enhanced report + timestamp = time.strftime("%Y%m%d-%H%M%S") + enhanced_filename = f"enhanced_security_report_{timestamp}.csv" + + # Update zip file with modified CSV + with zipfile.ZipFile(args.zip_file_name, 'a') as zout: + zout.writestr(enhanced_filename, output.getvalue()) + + logging.info(f"Enhanced security report saved to {args.zip_file_name}") + logging.info(f"Processed components: {processed_components}") + logging.info(f"Skipped components: {skipped_components}") + +def main(): + hub = HubInstance() + + project = hub.get_project_by_name(args.project_name) + + if project: + project_id = project['_meta']['href'].split('/')[-1] + logging.info(f"Project ID: {project_id}") + + version = hub.get_version_by_name(project, args.version_name) + if version: + project_version_id = version['_meta']['href'].split('/')[-1] + logging.info(f"Project Version ID: {project_version_id}") + + reports_l = [version_name_map.get(r.strip().lower(), r.strip()) for r in args.reports.split(",")] + + valid_reports = set(version_name_map.values()) + invalid_reports = [r for r in reports_l if r not in valid_reports] + if invalid_reports: + print(f"Error: Invalid report type(s): {', '.join(invalid_reports)}") + print(f"Valid report types are: {', '.join(valid_reports)}") + exit(1) + + response = hub.create_version_reports(version, reports_l, args.format) + + if response.status_code == 201: + print(f"Successfully created reports ({args.reports}) for project {args.project_name} and version {args.version_name}") + location = response.headers['Location'] + zip_content = download_report(location, args.zip_file_name) + + if 'SECURITY' in reports_l: + enhance_security_report(hub, zip_content, project_id, project_version_id) + else: + print(f"Failed to create reports for project {args.project_name} version {args.version_name}, status code returned {response.status_code}") + else: + print(f"Did not find version {args.version_name} for project {args.project_name}") + else: + print(f"Did not find project with name {args.project_name}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/generate_html_notices_report_from_json/requirements.lock.txt b/examples/generate_html_notices_report_from_json/requirements.lock.txt new file mode 100644 index 00000000..fba034f0 --- /dev/null +++ b/examples/generate_html_notices_report_from_json/requirements.lock.txt @@ -0,0 +1,97 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile examples/generate_html_notices_report_from_json/requirements.txt --generate-hashes -o examples/generate_html_notices_report_from_json/requirements.lock.txt +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via -r examples/generate_html_notices_report_from_json/requirements.txt +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 diff --git a/examples/get_project_version_total_scan_size.py b/examples/get_project_version_total_scan_size.py new file mode 100644 index 00000000..e29ab86f --- /dev/null +++ b/examples/get_project_version_total_scan_size.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +from blackduck.HubRestApi import HubInstance + +import argparse +import json + +parser = argparse.ArgumentParser() +parser.add_argument("project_name") +parser.add_argument("version_name") +args = parser.parse_args() + +hub = HubInstance() + +project = hub.get_project_by_name(args.project_name) +if project: + version = hub.get_version_by_name(project, args.version_name) + + if version: + codelocation_url = hub.get_link(version, "codelocations") + codelocation_url += "?limit={}".format(1000000) + response = hub.execute_get(codelocation_url) + if response.status_code == 200: + # codelocation and scan are synonymous + codelocation_info = response.json().get('items', []) + if codelocation_info: + most_recent_scan = max([cl['updatedAt'] for cl in codelocation_info]) + oldest_scan = min([cl['createdAt'] for cl in codelocation_info]) + number_scans = len(codelocation_info) + else: + number_scans = 0 + oldest_scan = most_recent_scan = "Not applicable" + + resultString = json.dumps( + { + 'scans': codelocation_info, + 'number_scans': number_scans, + 'most_recent_scan': most_recent_scan, + 'oldest_scan': oldest_scan + }) + resultArray = json.loads(resultString) + print(f"Number of scan performed {resultArray.get('number_scans')}") + scansArray = resultArray.get('scans') + sizeTrack=0 + for i, scan in enumerate(scansArray): + print(f"Scan {i+1} name:{scan['name']}") + sizeinbytes= scan['scanSize'] + size=sizeinbytes/1024/1024 + sizeTrack=sizeTrack+sizeinbytes + print(f" size:{'{0:.2f}'.format(size)} MB") + + totalsize=sizeTrack/1024/1024 + print("===============================") + print(f"Project Total size:{'{0:.2f}'.format(totalsize)} MB") + print("===============================") + + # print(json.dumps( + # { + # 'scans': codelocation_info, + # 'number_scans': number_scans, + # 'most_recent_scan': most_recent_scan, + # 'oldest_scan': oldest_scan + # })) + else: + print("Failed to retrieve the codelocation (aka scan) info, response code was {}".format(response.status_code)) + else: + print("Could not find the version {} in project {}".format(args.version_name, args.project_name)) +else: + print("Could not find the project {}".format(args.project_name)) \ No newline at end of file diff --git a/examples/scan_docker_image_lite.py b/examples/scan_docker_image_lite.py index e3c41cef..723e2075 100644 --- a/examples/scan_docker_image_lite.py +++ b/examples/scan_docker_image_lite.py @@ -5,7 +5,7 @@ Alternative version if Docker image layer by layer scan. -This program will download docker image and scan it into Blackduck server layer by layer +This program will download docker image and scan it into Black Duck server layer by layer Each layer will be scanned as a separate scan with a signature scan. Layers in the container images could be grouped into groups of contiguous layers. @@ -50,16 +50,16 @@ optional arguments: -h, --help show this help message and exit - --grouping GROUPING Group layers into user defined provect versions (can't be used with --base-image) + --grouping GROUPING Group layers into user defined project versions (can't be used with --base-image) --base-image BASE_IMAGE Use base image spec to determine base image/layers (can't be used with --grouping or --dockerfile) --dockerfile DOCKERFILE Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image) --project-name Specify project name (default is container image spec) - --project-verson Specify project version (default is container image tag/version) + --project-version Specify project version (default is container image tag/version) --detect-options DETECT_OPTIONS - Extra detect options to be passed directlyto the detect + Extra detect options to be passed directly to the detect Using --detect-options @@ -188,12 +188,25 @@ def read_config(self): with open(configFile) as fp: data = json.load(fp) return data + + def read_oci_layout(self): + oci_layout_file = self.imagedir + "/" + 'oci-layout' + if os.path.exists(oci_layout_file) and os.path.isfile(oci_layout_file): + with open(oci_layout_file) as fp: + data = json.load(fp) + return data + else: + return None + class Detector(): def __init__(self, hub): # self.detecturl = 'https://blackducksoftware.github.io/hub-detect/hub-detect.sh' # self.detecturl = 'https://detect.synopsys.com/detect.sh' - self.detecturl = 'https://detect.synopsys.com/detect7.sh' + # self.detecturl = 'https://detect.synopsys.com/detect7.sh' + # self.detecturl = 'https://detect.synopsys.com/detect8.sh' + self.detecturl = 'https://detect.blackduck.com/detect9.sh' + # self.detecturl = 'https://detect.blackduck.com/detect10.sh' self.baseurl = hub.config['baseurl'] self.filename = '/tmp/hub-detect.sh' self.token=hub.config['api_token'] @@ -240,6 +253,7 @@ def __init__( if detect_options: self.extra_options = detect_options.split(" ") print ("<--{}-->".format(self.grouping)) + self.binary = False def prepare_container_image(self): self.docker.initdir() @@ -262,6 +276,7 @@ def prepare_container_image(self): self.grouping = history_grouping self.docker.save_container_image(self.container_image_name) self.docker.unravel_container() + self.oci_layout = self.docker.read_oci_layout() def process_container_image_by_user_defined_groups(self): self.manifest = self.docker.read_manifest() @@ -333,25 +348,86 @@ def process_container_image_by_base_image_info(self): num = num + 1 print (json.dumps(self.layers, indent=4)) + def process_oci_container_image_by_user_defined_groups(self): + self.manifest = self.docker.read_manifest() + self.config = self.docker.read_config() + + self.layers = self.config['history'] + tagged_layers = [x for x in self.layers if '_group_end' in x.get('created_by')] + groups = {re.search('echo (.+?)_group_end', str(x['created_by'])).group(1): self.layers.index(x) for x in tagged_layers} + print (groups) + layer_paths = self.manifest[0]['Layers'].copy() + empty_layers = [x for x in self.layers if x.get('empty_layer', False)] + print(f"Total layers: {len(self.layers)} total paths: {len(layer_paths)} empty layers: {len(empty_layers)}") + + assert len(self.layers) == len(layer_paths) + len(empty_layers), "Something is wrong with this image, Layer math does not add up." + + for layer in self.layers: + layer['index'] = self.layers.index(layer) + if self.grouping: + layer['group_name'] = self.get_group_name(groups, layer['index']) + layer['project_version'] = "{}_{}".format(self.project_version,layer['group_name']) + layer['name'] = "{}_{}_{}_layer_{}".format(self.project_name,self.project_version,layer['group_name'],str(layer['index'])) + else: + layer['project_version'] = self.project_version + layer['name'] = self.project_name + "_" + self.project_version + "_layer_" + str(layer['index']) + layer['project_name'] = self.project_name + if not layer.get('empty_layer', False): + layer['path'] = layer_paths.pop(0) + print (json.dumps(self.layers, indent=4)) + + def get_group_name(self, groups, index): + group_name = 'undefined' + for group, value in groups.items(): + if index <= value: + group_name = group + break + return group_name + + def process_oci_container_image_by_base_image_info(self): + print ("Processing by BAse Image not supported for OCI images") + sys.exit(1) + pass + def process_container_image(self): + if self.oci_layout: + self.process_oci_container_image() + else: + self.process_docker_container_image() + + def process_docker_container_image(self): if self.grouping: self.process_container_image_by_user_defined_groups() else: self.process_container_image_by_base_image_info() + def process_oci_container_image(self): + if self.grouping: + self.process_oci_container_image_by_user_defined_groups() + else: + self.process_oci_container_image_by_base_image_info() + def submit_layer_scans(self): for layer in self.layers: - options = [] - options.append('--detect.project.name={}'.format(layer['project_name'])) - options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) - # options.append('--detect.blackduck.signature.scanner.disabled=false') - options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) - options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) - if self.base_image or self.grouping or self.dockerfile: - options.extend(self.adorn_extra_options(layer)) - else: - options.extend(self.extra_options) - self.hub_detect.detect_run(options) + if not layer.get('empty_layer', False): + options = [] + options.append('--detect.project.name={}'.format(layer['project_name'])) + options.append('--detect.project.version.name="{}"'.format(layer['project_version'])) + options.append('--detect.code.location.name={}_{}_code_{}'.format(layer['name'],self.image_version,layer['path'])) + if self.binary: + options.append('--detect.tools=BINARY_SCAN') + options.append('--detect.binary.scan.file.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.tools=SIGNATURE_SCAN') + if self.oci_layout: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'])) + else: + options.append('--detect.source.path={}/{}'.format(self.docker.imagedir, layer['path'].split('/')[0])) + if self.base_image or self.grouping or self.dockerfile: + options.extend(self.adorn_extra_options(layer)) + else: + options.extend(self.extra_options) + self.hub_detect.detect_run(options) def adorn_extra_options(self, layer): result = list() @@ -401,7 +477,7 @@ def get_base_layers(self): def scan_container_image( imagespec, grouping=None, base_image=None, dockerfile=None, - project_name=None, project_version=None, detect_options=None): + project_name=None, project_version=None, detect_options=None, binary=False): hub = HubInstance() scanner = ContainerImageScanner( @@ -416,6 +492,8 @@ def scan_container_image( scanner.grouping = '1024:everything' else: scanner.base_layers = scanner.get_base_layers() + if binary: + scanner.binary = True scanner.prepare_container_image() scanner.process_container_image() scanner.submit_layer_scans() @@ -429,12 +507,13 @@ def main(argv=None): parser = ArgumentParser() parser.add_argument('imagespec', help="Container image tag, e.g. repository/imagename:version") - parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined provect versions (can't be used with --base-image)") + parser.add_argument('--grouping',default=None, type=str, help="Group layers into user defined project versions (can't be used with --base-image)") parser.add_argument('--base-image',default=None, type=str, help="Use base image spec to determine base image/layers (can't be used with --grouping or --dockerfile)") parser.add_argument('--dockerfile',default=None, type=str, help="Use Dockerfile to determine base image/layers (can't be used with --grouping or ---base-image)") parser.add_argument('--project-name',default=None, type=str, help="Specify project name (default is container image spec)") parser.add_argument('--project-version',default=None, type=str, help="Specify project version (default is container image tag/version)") - parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directlyto the detect") + parser.add_argument('--detect-options',default=None, type=str, help="Extra detect options to be passed directly to the detect") + parser.add_argument('--binary', action='store_true', help="Use Binary Scan instead of signature scan") args = parser.parse_args() @@ -459,7 +538,8 @@ def main(argv=None): args.dockerfile, args.project_name, args.project_version, - args.detect_options) + args.detect_options, + args.binary) if __name__ == "__main__": diff --git a/examples/update_usergroups_from_csv.py b/examples/update_usergroups_from_csv.py index 7e2260e2..07df9de8 100644 --- a/examples/update_usergroups_from_csv.py +++ b/examples/update_usergroups_from_csv.py @@ -102,6 +102,8 @@ def update_user_group(hub_client, existing_name, new_name, user_group): # Update the name. user_group['name'] = new_name + #if user_group.get('externalGroupName') and user['externalGroupName'] == name: + user_group['externalName'] = new_name logging.info(f"Updating user group {existing_name} to {user_group['name']} for user group {user_group_url}") hub_client.session.put(user_group_url, json=user_group) diff --git a/examples/vuln_batch_remediation.py b/examples/vuln_batch_remediation.py index 2b006bb1..a6cfcb5e 100644 --- a/examples/vuln_batch_remediation.py +++ b/examples/vuln_batch_remediation.py @@ -84,7 +84,9 @@ def load_remediation_input(remediation_file): with open(remediation_file, mode='r', encoding="utf-8") as infile: - reader = csv.reader(infile) + dialect = csv.Sniffer().sniff(infile.read(), delimiters=';,') + infile.seek(0) + reader = csv.reader(infile, dialect) #return {rows[0]:[rows[1],rows[2]] for rows in reader} return {rows[0]:rows[1:] for rows in reader} @@ -95,7 +97,7 @@ def remediation_is_valid(vuln, remediation_data): if vulnerability_name in remediation_data.keys(): remediation = remediation_data[vulnerability_name] - if (remediation_status == remediation[0] and remediation_comment == remediation[1]): + if (remediation_status == remediation[0] and remediation_comment == remediation[1].replace('\\n','\n')): return None return remediation_data[vulnerability_name] else: @@ -127,11 +129,11 @@ def set_vulnerablity_remediation(hub, vuln, remediation_status, remediation_comm url = vuln['_meta']['href'] update={} update['remediationStatus'] = remediation_status - update['comment'] = remediation_comment + update['comment'] = remediation_comment.replace('\\n','\n') response = hub.execute_put(url, data=update) return response -def process_vulnerabilities(hub, vulnerable_components, remediation_data=None, exclusion_data=None, dry_run=False): +def process_vulnerabilities(hub, vulnerable_components, remediation_data=None, exclusion_data=None, dry_run=False, overwrite_existing=False): if (dry_run): print(f"Opening dry run output file: {dry_run}") @@ -142,8 +144,8 @@ def process_vulnerabilities(hub, vulnerable_components, remediation_data=None, e print('"Component Name","Component Version","CVE","Reason","Remeidation Status","HTTP response code"') for vuln in vulnerable_components['items']: - if vuln['vulnerabilityWithRemediation']['remediationStatus'] == "NEW": - remediation_action = None + if overwrite_existing or vuln['vulnerabilityWithRemediation']['remediationStatus'] == "NEW": + remediation_action = None exclusion_action = None if (remediation_data): @@ -164,8 +166,7 @@ def process_vulnerabilities(hub, vulnerable_components, remediation_data=None, e if (remediation_action): if (dry_run): - remediation_action.insert(0, vuln['vulnerabilityWithRemediation']['vulnerabilityName']) - csv_writer.writerow(remediation_action) + csv_writer.writerow([vuln['vulnerabilityWithRemediation']['vulnerabilityName']] + remediation_action) else: resp = set_vulnerablity_remediation(hub, vuln, remediation_action[0],remediation_action[1]) count += 1 @@ -218,6 +219,7 @@ def main(argv=None): # IGNORE:C0111 parser.add_argument("--cve-remediation-list-custom-field-label", default='CVE Remediation List', help='Label of Custom Field on Black Duck that contains remeidation list file name') parser.add_argument("--origin-exclusion-list-custom-field-label", default='Origin Exclusion List', help='Label of Custom Field on Black Duck that containts origin exclusion list file name') parser.add_argument('-V', '--version', action='version', version=program_version_message) + parser.add_argument("--overwrite-existing", dest='overwrite_existing', action="store_true", help='By default only NEW vulnerabilities are remediated. Enabling this flag will update all vulnerabilities.') # Process arguments args = parser.parse_args() @@ -231,6 +233,7 @@ def main(argv=None): # IGNORE:C0111 #dry_run = args.dry_run #dry_run_output = args.dry_run_output dry_run = args.dry_run + overwrite_existing = args.overwrite_existing print(args.dry_run) message = f"{program_version_message}\n\n Project: {projectname}\n Version: {projectversion}\n Process origin exclusion list: {process_origin_exclulsion}\n Process CVE remediation list: {process_cve_remediation}" @@ -271,11 +274,12 @@ def main(argv=None): # IGNORE:C0111 exclusion_data = None - # Retrieve the vulnerabiltites for the project version - vulnerable_components = hub.get_vulnerable_bom_components(version) - process_vulnerabilities(hub, vulnerable_components, remediation_data, exclusion_data, dry_run) - + # Retrieve the vulnerabiltites for the project version. Newer API versions only allow 1000 items at most. + vulnerable_components = hub.get_vulnerable_bom_components(version, 1000) + + process_vulnerabilities(hub, vulnerable_components, remediation_data, exclusion_data, dry_run, overwrite_existing) + return 0 except Exception: ### handle keyboard interrupt ### @@ -283,4 +287,4 @@ def main(argv=None): # IGNORE:C0111 return 0 if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/requirements.lock.txt b/requirements.lock.txt new file mode 100644 index 00000000..c18b3288 --- /dev/null +++ b/requirements.lock.txt @@ -0,0 +1,474 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --no-emit-index-url --no-emit-trusted-host --output-file=requirements.lock.txt --strip-extras requirements.txt +# +arrow==1.4.0 \ + --hash=sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205 \ + --hash=sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7 + # via -r requirements.txt +certifi==2026.5.20 \ + --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ + --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d + # via requests +charset-normalizer==3.4.7 \ + --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ + --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ + --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ + --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ + --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ + --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ + --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ + --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ + --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ + --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ + --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ + --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ + --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ + --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ + --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ + --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ + --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ + --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ + --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ + --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ + --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ + --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ + --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ + --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ + --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ + --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ + --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ + --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ + --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ + --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ + --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ + --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ + --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ + --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ + --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ + --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ + --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ + --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ + --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ + --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ + --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ + --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ + --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ + --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ + --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ + --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ + --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ + --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ + --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ + --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ + --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ + --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ + --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ + --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ + --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ + --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ + --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ + --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ + --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ + --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ + --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ + --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ + --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ + --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ + --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ + --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ + --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ + --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ + --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ + --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ + --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ + --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ + --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ + --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ + --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ + --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ + --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ + --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ + --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ + --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ + --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ + --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ + --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ + --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ + --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ + --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ + --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ + --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ + --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ + --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ + --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ + --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ + --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ + --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ + --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ + --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ + --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ + --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ + --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ + --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ + --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ + --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ + --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ + --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ + --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ + --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ + --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ + --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ + --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ + --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ + --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ + --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ + --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ + --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ + --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ + --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ + --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ + --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ + --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ + --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ + --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ + --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ + --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ + --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ + --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ + --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ + --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ + --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ + --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 + # via requests +docutils==0.23 \ + --hash=sha256:25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea \ + --hash=sha256:746f5060322511280a1e50eb76846ed6bf2342984b2ac04dc42caa1a8d78799e + # via readme-renderer +id==1.6.1 \ + --hash=sha256:279ec98b49c315880403d0e87b218e7d4e08c9c487992395a6b4498677042d47 \ + --hash=sha256:d0732d624fb46fd4e7bc4e5152f00214450953b9e772c182c1c22964def1a069 \ + --hash=sha256:f5ec41ed2629a508f5d0988eda142e190c9c6da971100612c4de9ad9f9b237ca + # via twine +idna==3.16 \ + --hash=sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5 \ + --hash=sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d + # via requests +iniconfig==2.3.0 \ + --hash=sha256:bd930604c1d2d3ef15b8cabe666358162e51874f80ad784aa61329c0d0bd8362 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +jaraco-classes==3.4.0 \ + --hash=sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd \ + --hash=sha256:828b4dde7f79f21d3ea9c933f393f6bdbd55e6303ed1cb1f2377874681939b6d \ + --hash=sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790 + # via keyring +jaraco-context==6.1.2 \ + --hash=sha256:7e9523faf738ebac6618a69251efeed20d4bbf25adf2add8eb0822278ca84c07 \ + --hash=sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535 \ + --hash=sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3 + # via keyring +jaraco-functools==4.5.0 \ + --hash=sha256:3bb5665ea4a020cf78a7040e89154c77edadb3ca74f366479669c5999aa70b03 \ + --hash=sha256:79ce39246eddbde4b3a03b77ea5f0f7878dc669b166a66cf3fa8e266aa3fa2f4 + # via keyring +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via pytest-html +keyring==25.7.0 \ + --hash=sha256:20c6e6b97380f809fb2f63f49b019cd04a221139319244c20758310104903974 \ + --hash=sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f \ + --hash=sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b + # via twine +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:746657e7635cdfa843d6ec4dc7ab87bd57ce5f13a9602ff58af2b57295842c92 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via jinja2 +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:93ad242e81180c5d119b52461ee4b364aeaab9454bdff49d4b411715d6f74f99 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +more-itertools==11.1.0 \ + --hash=sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d \ + --hash=sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 + # via + # jaraco-classes + # jaraco-functools +nh3==0.3.5 \ + --hash=sha256:005974514896b7e0cb9437876bebd7c6eb83b514a09c26cd74cae3d7c02e8ca4 \ + --hash=sha256:0a09f51806fd51b4fedbf9ea2b61fef388f19aef0d62fe51199d41648be14588 \ + --hash=sha256:0ef66a32c8fa957406e61b745d02f4b6dc9ad596f724a4c97aa37ea7bfb156c6 \ + --hash=sha256:1bb0ec0f3e9eea439f5f40dc241a7790cb8edd308bd48d31d85cf63dd68ba3d2 \ + --hash=sha256:207c01801d3e9bb8ec08f08689346bdd30ce15b8bf60013a925d08b5388962a4 \ + --hash=sha256:23a312224875f72cd16bde417f49071451877e29ef646a60e50fcb69407cc18a \ + --hash=sha256:2c069570b06aa848457713ad7af4a9905691291548c4466a9ad78ee95808382b \ + --hash=sha256:33c7793c1e5fbb8c332dabfda33fb7ee9c9257764575ffd2b727d253136c1fc0 \ + --hash=sha256:38748140bf76383ab7ce2dce0ad4cb663855d8fbc9098f7f3483673d09616a17 \ + --hash=sha256:387abd011e81959d5a35151a11350a0795c6edeb53ebfa02d2e882dc01299263 \ + --hash=sha256:3bb854485c9b33e5bb143ff3e49e577073bc6bc320f0ff8fc316dd89c0d3c101 \ + --hash=sha256:45855e14ff056064fec77133bfcf7cd691838168e5e17bbef075394954dc9dc8 \ + --hash=sha256:45e6a65dc88a300a2e3502cb9c8e6d1d6b831d6fba7470643333609c6aab1f30 \ + --hash=sha256:488928988caad25ba14b1eb5bc74e25e21f3b5e40341d956f3ce4a8bc19460dc \ + --hash=sha256:48f45e3e914be93a596431aa143dedf1582557bf41a58153c296048d6e3798c9 \ + --hash=sha256:505b57235390029a9608e98e493520540fc5349c70c94f19e39e1553b452bf44 \ + --hash=sha256:50d401ab2d8e86d59e2126e3ab2a2f45840c405842b626d9a51624b3a33b6878 \ + --hash=sha256:52d877980d7ca01dc3baf3936bf844828bc6f332962227a684ed79c18cce14c3 \ + --hash=sha256:53758dd9f91757e50fbbd0e34e269933af2b9f758351e35283acc06cf33d9afd \ + --hash=sha256:559e4c73b689e9a7aa97ac9760b1bc488038d7c1a575aa4ab5a0e19ee9630c0f \ + --hash=sha256:694b0b0b71871d04e0c24897d841dae9c007e97a653e48d68818e175007e1571 \ + --hash=sha256:6dfe85a907cf2b650312c75d7740f18ffb956dd8bc64eb7d7d625d4f265e9dd4 \ + --hash=sha256:6ea58cc44d274c643b83547ca9654a0b1a817609b160601356f76a2b744c49ad \ + --hash=sha256:72c5bdedec27fa33de6a5326346ea8aa3fe54f6ac294d54c4b204fb66a9f1e79 \ + --hash=sha256:7989e36f0170d44da9268ffda95935f91621c03f341f6e744b29deadee1b9c66 \ + --hash=sha256:84bdeb082544fbcb77a12c034dd77d7da0556fdc0727b787eb6214b958c15e29 \ + --hash=sha256:84e3047ddb57fe6692f321c9c556d4ce22405a84994ecedb4c3b682a09e5426d \ + --hash=sha256:869828201c6a4264133d5b24503b7301681566594f4ad37601c43c1cb4ac0b4c \ + --hash=sha256:8f85285700a18e9f3fc5bff41fe573fa84f81542ef13b48a89f9fecca0474d3b \ + --hash=sha256:937847a57a9cf5d86e1379a0dcccf16e72925a8df11c0ec67ec3ec92de7c9749 \ + --hash=sha256:94da8643b5aad56be378f3dc7be19f59be8e34877ecdc6dccc19ca2859efbac0 \ + --hash=sha256:a5e4a2dd9210e005fb4dd8a6d18c589c99002f540b59b03a22a321f1843c0ae6 \ + --hash=sha256:a9d63bdd9f0dabb0df4ee180b27837eaba0b04a49c721d8c8be168bff5bd4892 \ + --hash=sha256:acfd354e61accbe4c74f8017c6e397a776916dfe47c48643cf7fd84ade826f93 \ + --hash=sha256:bb9ae0bd69cbf2410691982341aced596492a37f12a733787999b8b74ad644e2 \ + --hash=sha256:bc579e7a5bfad44d7b9f2bb6b12fcdfb19124dc0b38b2d6aeafd5536da4e5c71 \ + --hash=sha256:c187635a0a39f5680d66a8226451d4fbbf3d905cca4646ef17e62a0f9d70cdc6 \ + --hash=sha256:c357f1d042c67f135a5e6babb2b0e3b9d9224ff4a3543240f597767b01384ffd \ + --hash=sha256:c3aae321f67ae66cff2a627115f106a377d4475d10b0e13d97959a13486b9a88 \ + --hash=sha256:c88605d8d468f7fc1b31e06129bc91d6c96f6c621776c9b504a0da9beac9df5f \ + --hash=sha256:c9ce78ce384fd4865cc0c0a662df34fc452ec75f4c94706c82d3996e9a395447 \ + --hash=sha256:c9d1d8dfb01a8dca2fe3aa758432baa006c0be32305c4f2f7198ce2a1537d244 \ + --hash=sha256:d74f4345cfef7298607af6d46db26dcd89885f49335159d38c36291b0ca16a8a \ + --hash=sha256:de8e8621853b6470fe928c684ee0d3f39ea8086cebafe4c416486488dea7b68d \ + --hash=sha256:e49c9b564e6bcb03ecd2f057213df9a0de15a95812ac9db9600b590db23d3ae9 \ + --hash=sha256:ea232933394d1d58bf7c4bb348dc4660eae6604e1ae81cd2ba6d9ed80d390f3b \ + --hash=sha256:eeedc90ed8c42c327e8e10e621ccfa314fc6cce35d5929f4297ff1cdb89667c4 \ + --hash=sha256:f5d471d436a6147c7a8075a2294f37f628318a5f7945489ddfa8605fb7641fc1 \ + --hash=sha256:fe3a787dc76b50de6bee54ef242f26c41dfe47654428e3e94f0fae5bb6dd2cc1 + # via readme-renderer +packaging==26.2 \ + --hash=sha256:41a77245620462d3c4fe6cdc2c61f01dee0e93180ce554120d92b0722575b3cf \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 + # via + # pytest + # twine + # wheel +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:dd599d9a0d5fbd9584da20ea661495e778e008401ce99f76963c1da2fc2c3f1d \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 \ + --hash=sha256:d3819ae5d79ed1f4347cd6baa51210b13ebcd6965b5e981a6cac3295acb63894 + # via + # pytest + # readme-renderer + # rich +pytest==9.0.3 \ + --hash=sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9 \ + --hash=sha256:b7e6b08772077d35e611ef43c30d186001e5b84c36210b4ff651acfb90f07408 \ + --hash=sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c + # via + # -r requirements.txt + # pytest-datadir + # pytest-html + # pytest-metadata +pytest-datadir==1.8.0 \ + --hash=sha256:5c677bc097d907ac71ca418109adc3abe34cf0bddfe6cf78aecfbabd96a15cf0 \ + --hash=sha256:7a15faed76cebe87cc91941dd1920a9a38eba56a09c11e9ddf1434d28a0f78eb + # via -r requirements.txt +pytest-html==4.2.0 \ + --hash=sha256:4dd34e9a332e96ba7fb140b68632258bec573d2c43166aa671a94986f72bbf7b \ + --hash=sha256:b6a88cba507500d8709959201e2e757d3941e859fd17cfd4ed87b16fc0c67912 \ + --hash=sha256:ff5caf3e17a974008e5816edda61168e6c3da442b078a44f8744865862a85636 + # via -r requirements.txt +pytest-metadata==3.1.1 \ + --hash=sha256:9011d5355e01c1e212b49faeba8831bffab9940d11fc833973fa556bf6fd7ccd \ + --hash=sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b \ + --hash=sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8 + # via pytest-html +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via + # -r requirements.txt + # arrow +pytz==2026.2 \ + --hash=sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126 \ + --hash=sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a \ + --hash=sha256:fa3d37e583c69b4a745dd92ad835f996407448e86c31c31825159ed8c36eb5fa + # via timestring +readme-renderer==44.0 \ + --hash=sha256:2fbca89b81a08526aadf1357a8c2ae889ec05fb03f5da67f9769c9a592166151 \ + --hash=sha256:8712034eabbfa6805cacf1402b4eeb2a73028f72d1166d6f5cb7f9c047c5d1e1 \ + --hash=sha256:950609fbe5e9d9b6ae18770a2b896cbad7194ee763377bdf10b22ee910c9df40 + # via twine +requests==2.32.4 \ + --hash=sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c \ + --hash=sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422 \ + --hash=sha256:acc14812958ea21ca50b8bab45099264b3d7feba361acf40ddafd78832b05b98 + # via + # -r requirements.txt + # requests-mock + # requests-toolbelt + # twine +requests-mock==1.12.1 \ + --hash=sha256:3e8db63dff19719391b8c7c678d9e973e582361eeab0b3f80fa9228add556ca6 \ + --hash=sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563 \ + --hash=sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401 + # via -r requirements.txt +requests-toolbelt==1.0.0 \ + --hash=sha256:392594a42d9a9f291b9f04af37559fceadac8d3f8ce1ff6045f4922d36426c32 \ + --hash=sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6 \ + --hash=sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06 + # via twine +rfc3986==2.0.0 \ + --hash=sha256:482a9b7950ab1c8458d321d4d9fc62e9e7e2bf5982378d552d4d36a950e9cc5b \ + --hash=sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd \ + --hash=sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c + # via twine +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:5dd8ecaf5d0da3119949338c3a4a70894a82c2509f6acd0f7d6916dbc36bd24f \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via twine +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +terminaltables==3.1.10 \ + --hash=sha256:ba6eca5cb5ba02bba4c9f4f985af80c54ec3dccf94cfcd190154386255e47543 \ + --hash=sha256:e4fdc4179c9e4aab5f674d80f09d76fa436b96fdc698a8505e0a36bf0804a874 + # via -r requirements.txt +timestring==1.6.4 \ + --hash=sha256:5c80d8cc555fd31d4d743acb4f028c9ddea294b247cdb407afb7ca050d5c4216 \ + --hash=sha256:b7d1a2060a5f2e34d0c93b1042690074afcea66638fa2cf4dcad2d1bec4deacc + # via -r requirements.txt +twine==6.2.0 \ + --hash=sha256:3da303f52d63484e824bfb792df1a2c92e849c6d5e1feffc7fb81c71def4e924 \ + --hash=sha256:418ebf08ccda9a8caaebe414433b0ba5e25eb5e4a927667122fbe8f829f985d8 \ + --hash=sha256:e5ed0d2fd70c9959770dce51c8f39c8945c574e18173a7b81802dab51b4b75cf + # via -r requirements.txt +tzdata==2026.2 \ + --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \ + --hash=sha256:93e00b2b8e83bd91fa05a073dc9d71a565e9c3c9eef0889ebedd978dab0c5f08 \ + --hash=sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7 + # via arrow +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + # via + # id + # requests + # twine +wheel==0.47.0 \ + --hash=sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced \ + --hash=sha256:66c570b5f1e7276af684b2d1f11070dd332e0d2b78d8f649757d377cd0b211b0 \ + --hash=sha256:cc72bd1009ba0cf63922e28f94d9d83b920aa2bb28f798a31d0691b02fa3c9b3 + # via -r requirements.txt + +# The following packages are considered to be unsafe in a requirements file: +setuptools==82.0.1 \ + --hash=sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9 \ + --hash=sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb \ + --hash=sha256:f2f48a947374136a39e9f349cf28f1900765657b892322e7ead81d0f466db134 + # via -r requirements.txt diff --git a/requirements.txt b/requirements.txt index 6ee96357..69c138a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # For the library itself -requests==2.31.0 -python-dateutil>=2.8.0 +requests==2.32.4 +python-dateutil==2.9.0.post0 # for examples printing tables to the terminal terminaltables diff --git a/setup.py b/setup.py index 7eadcc73..51ad8d13 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,11 @@ 'requests', 'python-dateutil' ] +# Optional dependencies for MCP server +EXTRAS = { + 'mcp': ['fastmcp'] +} + # The rest you shouldn't have to touch too much :) # ------------------------------------------------ # Except, perhaps the License and Trove Classifiers! @@ -100,9 +105,9 @@ def run(self): # If your package is a single module, use this instead of 'packages': # py_modules=['mypackage'], - # entry_points={ - # 'console_scripts': ['mycli=mymodule:cli'], - # }, + entry_points={ + 'console_scripts': ['blackduck=blackduck.__main__:main'], + }, # Seems like dependency_links is potentiall going away. That said, using pip 10 # I was able to install using 'pip install dist/hubpy-0.0.1.tar.gz --process-dependency-links' @@ -111,6 +116,7 @@ def run(self): # 'git+git://github.com/blackducksoftware/tortilla#egg=tortilla-0.5.1b' # ], install_requires=REQUIRED, + extras_require=EXTRAS, setup_requires=['pytest-runner'], tests_require=['pytest', 'requests-mock', 'pytest-datadir'], include_package_data=True,