From a59382406896c26760d09ff24f901d1b40ac0ad7 Mon Sep 17 00:00:00 2001 From: ljha Date: Tue, 6 Oct 2020 21:50:00 +0530 Subject: [PATCH 1/3] FAB-30 - Added new command `cortex deploy` to export Cortex artifacts for CI deployment --- bin/cortex-deploy.js | 47 ++++++++++++++++++++++ bin/cortex.js | 1 + src/commands/deploy.js | 91 ++++++++++++++++++++++++++++++++++++++++++ src/commands/utils.js | 29 +++++++++++++- 4 files changed, 166 insertions(+), 2 deletions(-) create mode 100755 bin/cortex-deploy.js create mode 100644 src/commands/deploy.js diff --git a/bin/cortex-deploy.js b/bin/cortex-deploy.js new file mode 100755 index 00000000..095d12e5 --- /dev/null +++ b/bin/cortex-deploy.js @@ -0,0 +1,47 @@ +#!/usr/bin/env node + +/* + * Copyright 2018 Cognitive Scale, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const chalk = require('chalk'); +const program = require('../src/commander'); + +const { withCompatibilityCheck } = require('../src/compatibility'); + +const { + DeploySnapshotCommand +} = require('../src/commands/deploy'); + +program.description('Export Cortex artifacts for deployment'); + +// Export Agent Snapshot. Provide snapshotIds in quotes separated by space like "snapshotId1 snapshotId2". This is to support multiple agents in one deployment manifest file. +program + .command('snapshots ') + .description('Export Agent(s) snapshots(s) for deployment. Provide snapshotIds separated by space, as <"snapshotId1 snapshotId2 ...">') + .option('--no-compat', 'Ignore API compatibility checks') + .option('--color [on/off]', 'Turn on/off colors for JSON output.', 'on') + .option('--profile [profile]', 'The profile to use') + .option('-y, --yaml', 'Use YAML for snapshot export format') + .action(withCompatibilityCheck((skillDefinition, options) => { + try { + new DeploySnapshotCommand(program).execute(skillDefinition, options); + } + catch (err) { + console.error(chalk.red(err.message)); + } + })); + +program.parse(process.argv); diff --git a/bin/cortex.js b/bin/cortex.js index f27ca511..9bd707bb 100755 --- a/bin/cortex.js +++ b/bin/cortex.js @@ -33,6 +33,7 @@ program .command('connections [cmd]', 'Work with Cortex Connections') .command('content [cmd]', 'Work with Cortex Managed Content') .command('environments [cmd]', 'Work with Cortex Environments') + .command('deploy [cmd]', 'Work with Cortex Artifacts export for deployment') .command('datasets [cmd]', 'Work with Cortex Datasets') .command('docker [cmd]', 'Work with Docker') .command('jobs [cmd]', 'Work with Cortex v2 Jobs') diff --git a/src/commands/deploy.js b/src/commands/deploy.js new file mode 100644 index 00000000..17dc84b3 --- /dev/null +++ b/src/commands/deploy.js @@ -0,0 +1,91 @@ +/* + * Copyright 2018 Cognitive Scale, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the “License”); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +const debug = require('debug')('cortex:cli'); +const { loadProfile } = require('../config'); +const Agents = require('../client/agents'); +const { printSuccess, printError, filterObject, cleanInternalFields, jsonToYaml, writeToFile } = require('./utils'); + + +/** + * Cortex deploy command is to export Cortex artifacts (agent, snapshot, skill, action etc) for CI/CD deployment. This command: + * 1. Exports cortex artifacts + * 2. generates manifest files to drive artifacts deployment + * Files are exported/generated in pre-determined layout to support CI/CD deployment + * .fabric + * snapshots + * snapshot1.yaml + * snapshot2.yaml + * agents + * ... + * skills + * ... + * actions + * ... + * fabric.yaml (manifest file) + * + * Currently only exporting agent snapshots. This need to be updated for v6. + * + * @type {DeploySnapshotCommand} + */ +module.exports.DeploySnapshotCommand = class { + constructor(program) { + this.program = program; + } + + execute(snapshotIds, options) { + const exportPath = '.fabric/snapshots/'; + const manifestFile = 'fabric.yaml'; + + const profile = loadProfile(options.profile); + const envName = options.environmentName; + debug('%s.exportDeploymentSnapshot(%s)', profile.name, snapshotIds); + + const agents = new Agents(profile.url); + const promises = []; + snapshotIds.split(' ').forEach(function (snapshotId) { + promises.push(agents.describeAgentSnapshot(profile.token, snapshotId, envName).then((response) => { + if (response.success) { + let result = filterObject(response.result, options); + result = cleanInternalFields(result); + let filename = snapshotId+".json"; + if (options.yaml) { + result = jsonToYaml(result); + filename = snapshotId+".yaml"; + } + const filepath = exportPath + filename; + writeToFile(result, filepath); + printSuccess(`Successfully exported agent snapshot ${filepath}`); + return filepath; + } else { + printError(`Failed to export agent snapshot ${snapshotId}: ${response.message}`, options); + } + }).catch((err) => { + printError(`Failed to export agent snapshot ${snapshotId}: ${err.status} ${err.message}`, options); + })); + }); + + Promise.all(promises).then(result => { + const manifest = { + "version": 1, + "cortex": { + "snapshots": result + } + }; + writeToFile(jsonToYaml(manifest), manifestFile); + printSuccess(`Successfully generated manifest file ${manifestFile}`); + }); + } +}; diff --git a/src/commands/utils.js b/src/commands/utils.js index 6da9413d..15bb8fbb 100644 --- a/src/commands/utils.js +++ b/src/commands/utils.js @@ -84,7 +84,7 @@ module.exports.parseObject = function(str, options) { if (options.yaml) { return yaml.safeLoad(str); } - + return JSON.parse(str); }; @@ -195,7 +195,7 @@ module.exports.formatAllServiceInputParameters = function(allParameters){ return `$ref:${allParameters.$ref}`; } else{ - return allParameters.map(inputParameters => formatServiceInputParameter(inputParameters)).join('\n'); + return allParameters.map(inputParameters => formatServiceInputParameter(inputParameters)).join('\n'); } } @@ -246,3 +246,28 @@ module.exports.formatValidationPath = (p) => { }); return res; }; + +module.exports.cleanInternalFields = function(jsobj) { + return JSON.stringify(jsobj, function re(a, obj) { + if (a.startsWith("_")) { + return undefined; + } + return obj; + }, 2); +}; + +module.exports.jsonToYaml = function (json) { + if (typeof json == 'string') { + json = JSON.parse(json); + } + return yaml.dump(json); +}; + +module.exports.writeToFile = function (content, filepath) { + const dir = path.dirname(filepath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, {recursive: true}); + } + + fs.writeFileSync(filepath, content); +}; From 7e68c86311e2d9e19f0ca1ad0061e5f401d8a55f Mon Sep 17 00:00:00 2001 From: ljha Date: Wed, 7 Oct 2020 19:08:16 +0530 Subject: [PATCH 2/3] FAB-30 - Added new command `cortex deploy` to export Cortex artifacts for CI deployment --- src/commands/deploy.js | 7 ++++++- src/commands/utils.js | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/commands/deploy.js b/src/commands/deploy.js index 17dc84b3..5217843b 100644 --- a/src/commands/deploy.js +++ b/src/commands/deploy.js @@ -16,7 +16,7 @@ const debug = require('debug')('cortex:cli'); const { loadProfile } = require('../config'); const Agents = require('../client/agents'); -const { printSuccess, printError, filterObject, cleanInternalFields, jsonToYaml, writeToFile } = require('./utils'); +const { printSuccess, printError, filterObject, cleanInternalFields, jsonToYaml, writeToFile, fileExists } = require('./utils'); /** @@ -49,6 +49,10 @@ module.exports.DeploySnapshotCommand = class { const exportPath = '.fabric/snapshots/'; const manifestFile = 'fabric.yaml'; + if (fileExists(exportPath) || fileExists(manifestFile)) { + printError(`Export path ${exportPath} or manifest file ${manifestFile} already exists`) + } + const profile = loadProfile(options.profile); const envName = options.environmentName; debug('%s.exportDeploymentSnapshot(%s)', profile.name, snapshotIds); @@ -60,6 +64,7 @@ module.exports.DeploySnapshotCommand = class { if (response.success) { let result = filterObject(response.result, options); result = cleanInternalFields(result); + //TODO add validation for not exporting tip snapshots, as they don't have all dependencies let filename = snapshotId+".json"; if (options.yaml) { result = jsonToYaml(result); diff --git a/src/commands/utils.js b/src/commands/utils.js index 15bb8fbb..3b706b51 100644 --- a/src/commands/utils.js +++ b/src/commands/utils.js @@ -271,3 +271,7 @@ module.exports.writeToFile = function (content, filepath) { fs.writeFileSync(filepath, content); }; + +module.exports.fileExists = function (filepath) { + return fs.existsSync(filepath) +}; From f115804130fa3a12df9270e620a8eeb849e844ac Mon Sep 17 00:00:00 2001 From: ljha Date: Fri, 9 Oct 2020 12:24:09 +0530 Subject: [PATCH 3/3] added --- bin/cortex-deploy.js | 1 + src/commands/deploy.js | 45 ++++++++++++++++++++++++------------------ src/commands/utils.js | 25 +++++++++++++++++++++++ 3 files changed, 52 insertions(+), 19 deletions(-) diff --git a/bin/cortex-deploy.js b/bin/cortex-deploy.js index 095d12e5..d9bc8505 100755 --- a/bin/cortex-deploy.js +++ b/bin/cortex-deploy.js @@ -35,6 +35,7 @@ program .option('--color [on/off]', 'Turn on/off colors for JSON output.', 'on') .option('--profile [profile]', 'The profile to use') .option('-y, --yaml', 'Use YAML for snapshot export format') + .option('-f, --force', 'Force delete existing exported files') .action(withCompatibilityCheck((skillDefinition, options) => { try { new DeploySnapshotCommand(program).execute(skillDefinition, options); diff --git a/src/commands/deploy.js b/src/commands/deploy.js index 5217843b..a4cfbc1c 100644 --- a/src/commands/deploy.js +++ b/src/commands/deploy.js @@ -14,9 +14,10 @@ * limitations under the License. */ const debug = require('debug')('cortex:cli'); -const { loadProfile } = require('../config'); +const {loadProfile} = require('../config'); +const path = require('path'); const Agents = require('../client/agents'); -const { printSuccess, printError, filterObject, cleanInternalFields, jsonToYaml, writeToFile, fileExists } = require('./utils'); +const {printSuccess, printError, filterObject, cleanInternalFields, jsonToYaml, writeToFile, fileExists, deleteFile} = require('./utils'); /** @@ -36,7 +37,7 @@ const { printSuccess, printError, filterObject, cleanInternalFields, jsonToYaml, * ... * fabric.yaml (manifest file) * - * Currently only exporting agent snapshots. This need to be updated for v6. + * Currently only exporting agent snapshots. * * @type {DeploySnapshotCommand} */ @@ -46,13 +47,18 @@ module.exports.DeploySnapshotCommand = class { } execute(snapshotIds, options) { - const exportPath = '.fabric/snapshots/'; + const exportPath = '.fabric'; const manifestFile = 'fabric.yaml'; if (fileExists(exportPath) || fileExists(manifestFile)) { - printError(`Export path ${exportPath} or manifest file ${manifestFile} already exists`) + if (options.force) { + deleteFile(exportPath); + deleteFile(manifestFile); + printSuccess(`Deleted ${exportPath} and ${manifestFile}`); + } else { + printError(`Aborting, because export path ${exportPath} or manifest file ${manifestFile} exists. Use -f option to force delete.`); + } } - const profile = loadProfile(options.profile); const envName = options.environmentName; debug('%s.exportDeploymentSnapshot(%s)', profile.name, snapshotIds); @@ -62,29 +68,30 @@ module.exports.DeploySnapshotCommand = class { snapshotIds.split(' ').forEach(function (snapshotId) { promises.push(agents.describeAgentSnapshot(profile.token, snapshotId, envName).then((response) => { if (response.success) { - let result = filterObject(response.result, options); - result = cleanInternalFields(result); - //TODO add validation for not exporting tip snapshots, as they don't have all dependencies - let filename = snapshotId+".json"; - if (options.yaml) { - result = jsonToYaml(result); - filename = snapshotId+".yaml"; - } - const filepath = exportPath + filename; - writeToFile(result, filepath); - printSuccess(`Successfully exported agent snapshot ${filepath}`); - return filepath; + let result = filterObject(response.result, options); + result = cleanInternalFields(result); + //TODO add validation for not exporting tip snapshots, as they don't have all dependencies. Should we enforce? + let filename = snapshotId + ".json"; + if (options.yaml) { + result = jsonToYaml(result); + filename = snapshotId + ".yaml"; + } + const filepath = path.join(exportPath, 'snapshots', filename); + writeToFile(result, filepath); + printSuccess(`Successfully exported agent snapshot ${filepath}`); + return filepath; } else { printError(`Failed to export agent snapshot ${snapshotId}: ${response.message}`, options); } }).catch((err) => { - printError(`Failed to export agent snapshot ${snapshotId}: ${err.status} ${err.message}`, options); + printError(`Failed to export agent snapshot ${snapshotId}: ${err.status} ${err.message}`, options); })); }); Promise.all(promises).then(result => { const manifest = { "version": 1, + "kind": "deployment-manifest", "cortex": { "snapshots": result } diff --git a/src/commands/utils.js b/src/commands/utils.js index 3b706b51..7d7d8385 100644 --- a/src/commands/utils.js +++ b/src/commands/utils.js @@ -275,3 +275,28 @@ module.exports.writeToFile = function (content, filepath) { module.exports.fileExists = function (filepath) { return fs.existsSync(filepath) }; + +// Alternatively, we can use fs.rmdirSync(, {recursive: true}), but that requires node v12+ +const deleteFolderRecursive = function(filepath) { + try { + if (fs.existsSync(filepath)) { + if (fs.lstatSync(filepath).isDirectory()) { + fs.readdirSync(filepath).forEach((file, index) => { + const curPath = path.join(filepath, file); + if (fs.lstatSync(curPath).isDirectory()) { // recurse + deleteFolderRecursive(curPath); + } else { // delete file + fs.unlinkSync(curPath); + } + }); + fs.rmdirSync(filepath); + } else { + fs.unlinkSync(filepath); + } + } + } catch (e) { + console.error(chalk.red(e.message)); + } +}; + +module.exports.deleteFile = deleteFolderRecursive;