import * as child_process from 'child_process'; import * as fs from 'fs'; import * as zlib from 'zlib'; import * as url from 'url'; import * as http from 'http'; import * as https from 'https'; import * as events from 'events'; import * as crypto from 'crypto'; import * as path from 'path'; import * as os from 'os'; Promise = require("bluebird"); import Util = pxt.Util; export interface SpawnOptions { cmd: string; args: string[]; cwd?: string; shell?: boolean; pipe?: boolean; input?: string; silent?: boolean; envOverrides?: pxt.Map; allowNonZeroExit?: boolean; } //This should be correct at startup when running from command line export let targetDir: string = process.cwd(); export let pxtCoreDir: string = path.join(__dirname, ".."); export let cliFinalizers: (() => Promise)[] = []; export function addCliFinalizer(f: () => Promise) { cliFinalizers.push(f) } export function runCliFinalizersAsync() { let fins = cliFinalizers cliFinalizers = [] return Promise.mapSeries(fins, f => f()) .then(() => { }) } export function setTargetDir(dir: string) { targetDir = dir; (module).paths.push(path.join(targetDir, "node_modules")); } export function readResAsync(g: events.EventEmitter) { return new Promise((resolve, reject) => { let bufs: Buffer[] = [] g.on('data', (c: any) => { if (typeof c === "string") bufs.push(Buffer.from(c, "utf8")) else bufs.push(c) }); g.on("error", (err: any) => reject(err)) g.on('end', () => resolve(Buffer.concat(bufs))) }) } export function spawnAsync(opts: SpawnOptions) { opts.pipe = false return spawnWithPipeAsync(opts) .then(() => { }) } export function spawnWithPipeAsync(opts: SpawnOptions) { if (opts.pipe === undefined) opts.pipe = true let info = opts.cmd + " " + opts.args.join(" ") if (opts.cwd && opts.cwd != ".") info = "cd " + opts.cwd + "; " + info console.log("[run] " + info) return new Promise((resolve, reject) => { let ch = child_process.spawn(opts.cmd, opts.args, { cwd: opts.cwd, env: opts.envOverrides ? extendEnv(process.env, opts.envOverrides) : process.env, stdio: opts.pipe ? [opts.input == null ? process.stdin : "pipe", "pipe", process.stderr] : "inherit", shell: opts.shell || false } as any) let bufs: Buffer[] = [] if (opts.pipe) ch.stdout.on('data', (buf: Buffer) => { bufs.push(buf) if (!opts.silent) { process.stdout.write(buf) } }) ch.on('close', (code: number) => { if (code != 0 && !opts.allowNonZeroExit) reject(new Error("Exit code: " + code + " from " + info)) resolve(Buffer.concat(bufs)) }); if (opts.input != null) ch.stdin.end(opts.input, "utf8") }) } function extendEnv(base: any, overrides: any) { let res: any = {}; Object.keys(base).forEach(key => res[key] = base[key]) Object.keys(overrides).forEach(key => res[key] = overrides[key]) return res; } export function addCmd(name: string) { return name + (/^win/.test(process.platform) ? ".cmd" : "") } export function runNpmAsync(...args: string[]) { return runNpmAsyncWithCwd(".", ...args); } export interface NpmRegistry { _id: string; _name: string; "dist-tags": pxt.Map; "versions": pxt.Map; } export function npmRegistryAsync(pkg: string): Promise { // TODO: use token if available return Util.httpGetJsonAsync(`https://registry.npmjs.org/${pkg}`); } export function runNpmAsyncWithCwd(cwd: string, ...args: string[]) { return spawnAsync({ cmd: addCmd("npm"), args: args, cwd }); } export function runGitAsync(...args: string[]) { return spawnAsync({ cmd: "git", args: args, cwd: "." }) } export function gitInfoAsync(args: string[], cwd?: string, silent: boolean = false) { return Promise.resolve() .then(() => spawnWithPipeAsync({ cmd: "git", args: args, cwd, silent })) .then(buf => buf.toString("utf8").trim()) } export function currGitTagAsync() { return gitInfoAsync(["describe", "--tags", "--exact-match"]) .then(t => { if (!t) Util.userError("no git tag found") return t }) } export function needsGitCleanAsync() { return Promise.resolve() .then(() => spawnWithPipeAsync({ cmd: "git", args: ["status", "--porcelain", "--untracked-files=no"] })) .then(buf => { if (buf.length) Util.userError("Please commit all files to git before running 'pxt bump'") }) } function nodeHttpRequestAsync(options: Util.HttpRequestOptions): Promise { let isHttps = false let u = url.parse(options.url) if (u.protocol == "https:") isHttps = true /* tslint:disable:no-http-string */ else if (u.protocol == "http:") isHttps = false /* tslint:enable:no-http-string */ else return Promise.reject("bad protocol: " + u.protocol) u.headers = Util.clone(options.headers) || {} let data = options.data u.method = options.method || (data == null ? "GET" : "POST"); let buf: Buffer = null; u.headers["accept-encoding"] = "gzip" u.headers["user-agent"] = "PXT-CLI" let gzipContent = false if (data != null) { if (Buffer.isBuffer(data)) { buf = data; } else if (typeof data == "object") { buf = Buffer.from(JSON.stringify(data), "utf8") u.headers["content-type"] = "application/json; charset=utf8" if (options.allowGzipPost) gzipContent = true } else if (typeof data == "string") { buf = Buffer.from(data, "utf8") if (options.allowGzipPost) gzipContent = true } else { Util.oops("bad data") } } if (gzipContent) { buf = zlib.gzipSync(buf) u.headers['content-encoding'] = "gzip" } if (buf) u.headers['content-length'] = buf.length return new Promise((resolve, reject) => { const handleResponse = (res: http.IncomingMessage) => { let g: events.EventEmitter = res; if (/gzip/.test(res.headers['content-encoding'])) { let tmp = zlib.createUnzip(); res.pipe(tmp); g = tmp; } resolve(readResAsync(g).then(buf => { let text: string = null try { text = buf.toString("utf8") } catch (e) { } let resp: Util.HttpResponse = { statusCode: res.statusCode, headers: res.headers, buffer: buf, text: text } return resp; })) }; const req = isHttps ? https.request(u, handleResponse) : http.request(u, handleResponse); req.on('error', (err: any) => reject(err)) req.end(buf) }) } function sha256(hashData: string): string { let sha: string; let hash = crypto.createHash("sha256"); hash.update(hashData, "utf8"); sha = hash.digest().toString("hex").toLowerCase(); return sha; } function init() { // no, please, I want to handle my errors myself let async = (Promise)._async async.fatalError = (e: any) => async.throwLater(e); Util.isNodeJS = true; Util.httpRequestCoreAsync = nodeHttpRequestAsync; Util.sha256 = sha256; Util.cpuUs = () => { const p = process.cpuUsage() return p.system + p.user } Util.getRandomBuf = buf => { let tmp = crypto.randomBytes(buf.length) for (let i = 0; i < buf.length; ++i) buf[i] = tmp[i] } (global as any).btoa = (str: string) => Buffer.from(str, "binary").toString("base64"); (global as any).atob = (str: string) => Buffer.from(str, "base64").toString("binary"); } export function sanitizePath(path: string) { return path.replace(/[^\w@\/]/g, "-").replace(/^\/+/, "") } export function readJson(fn: string) { return JSON.parse(fs.readFileSync(fn, "utf8")) } export function readText(fn: string) { return fs.readFileSync(fn, "utf8"); } export function readPkgConfig(dir: string) { //pxt.debug("readPkgConfig in " + dir) const fn = path.join(dir, pxt.CONFIG_NAME) const js: pxt.PackageConfig = readJson(fn) const ap = js.additionalFilePath if (ap) { let adddir = path.join(dir, ap); if (!existsDirSync(adddir)) pxt.U.userError(`additional pxt.json not found: ${adddir} in ${dir} + ${ap}`) pxt.debug("additional pxt.json: " + adddir) const js2 = readPkgConfig(adddir) for (let k of Object.keys(js2)) { if (!js.hasOwnProperty(k)) { (js as any)[k] = (js2 as any)[k] } } js.additionalFilePaths = [ap].concat(js2.additionalFilePaths.map(d => path.join(ap, d))) } else { js.additionalFilePaths = [] } // don't inject version number // as they get serialized later on // if (!js.targetVersions) js.targetVersions = pxt.appTarget.versions; return js } export function getPxtTarget(): pxt.TargetBundle { if (fs.existsSync(targetDir + "/built/target.json")) { let res: pxt.TargetBundle = readJson(targetDir + "/built/target.json") if (res.id && res.bundledpkgs) return res; } let raw: pxt.TargetBundle = readJson(targetDir + "/pxtarget.json") raw.bundledpkgs = {} return raw } export function pathToPtr(path: string) { return "ptr-" + sanitizePath(path.replace(/^ptr-/, "")).replace(/[^\w@]/g, "-") } export function mkdirP(thePath: string) { if (thePath == "." || !thePath) return; if (!fs.existsSync(thePath)) { mkdirP(path.dirname(thePath)) fs.mkdirSync(thePath) } } export function cpR(src: string, dst: string, maxDepth = 8) { src = path.resolve(src) let files = allFiles(src, maxDepth) let dirs: pxt.Map = {} for (let f of files) { let bn = f.slice(src.length) let dd = path.join(dst, bn) let dir = path.dirname(dd) if (!Util.lookup(dirs, dir)) { mkdirP(dir) dirs[dir] = true } let buf = fs.readFileSync(f) fs.writeFileSync(dd, buf) } } export function cp(srcFile: string, destDirectory: string) { mkdirP(destDirectory); let dest = path.resolve(destDirectory, path.basename(srcFile)); let buf = fs.readFileSync(path.resolve(srcFile)); fs.writeFileSync(dest, buf); } export function allFiles(top: string, maxDepth = 8, allowMissing = false, includeDirs = false, ignoredFileMarker: string = undefined): string[] { let res: string[] = [] if (allowMissing && !existsDirSync(top)) return res for (const p of fs.readdirSync(top)) { if (p[0] == ".") continue; const inner = path.join(top, p) const st = fs.statSync(inner) if (st.isDirectory()) { // check for ingored folder marker if (ignoredFileMarker && fs.existsSync(path.join(inner, ignoredFileMarker))) continue; if (maxDepth > 1) Util.pushRange(res, allFiles(inner, maxDepth - 1)) if (includeDirs) res.push(inner); } else { res.push(inner) } } return res } export function existsDirSync(name: string): boolean { try { const stats = fs.lstatSync(name); return stats && stats.isDirectory(); } catch (e) { return false; } } export function writeFileSync(p: string, data: any, options?: { encoding?: string | null; mode?: number | string; flag?: string; } | string | null) { mkdirP(path.dirname(p)); fs.writeFileSync(p, data, options); if (pxt.options.debug) { const stats = fs.statSync(p); pxt.log(` + ${p} ${stats.size > 1000000 ? (stats.size / 1000000).toFixed(2) + ' m' : stats.size > 1000 ? (stats.size / 1000).toFixed(2) + 'k' : stats.size}b`) } } export function openUrl(startUrl: string, browser: string) { if (!/^[a-z0-9A-Z#=\.\-\\\/%:\?_&]+$/.test(startUrl)) { console.error("invalid URL to open: " + startUrl) return } let cmds: pxt.Map = { darwin: "open", win32: "start", linux: "xdg-open" } if (/^win/.test(os.platform()) && !/^[a-z0-9]+:\/\//i.test(startUrl)) startUrl = startUrl.replace('/', '\\'); else startUrl = startUrl.replace('\\', '/'); console.log(`opening ${startUrl}`) if (browser) { child_process.spawn(getBrowserLocation(browser), [startUrl], { detached: true }); } else { child_process.exec(`${cmds[process.platform]} ${startUrl}`); } } function getBrowserLocation(browser: string) { let browserPath: string; const normalizedBrowser = browser.toLowerCase(); if (normalizedBrowser === "chrome") { switch (os.platform()) { case "win32": browserPath = "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe"; break; case "darwin": browserPath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"; break; case "linux": browserPath = "/opt/google/chrome/chrome"; break; default: break; } } else if (normalizedBrowser === "firefox") { browserPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe"; switch (os.platform()) { case "win32": browserPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe"; break; case "darwin": browserPath = "/Applications/Firefox.app"; break; case "linux": default: break; } } else if (normalizedBrowser === "ie") { browserPath = "C:/Program Files/Internet Explorer/iexplore.exe"; } else if (normalizedBrowser === "safari") { browserPath = "/Applications/Safari.app/Contents/MacOS/Safari"; } if (browserPath && fs.existsSync(browserPath)) { return browserPath; } return browser; } export function fileExistsSync(p: string): boolean { try { let stats = fs.lstatSync(p); return stats && stats.isFile(); } catch (e) { return false; } } export let lastResolveMdDirs: string[] = [] // returns undefined if not found export function resolveMd(root: string, pathname: string, md?: string): string { const docs = path.join(root, "docs"); const tryRead = (fn: string) => { if (fileExistsSync(fn + ".md")) return fs.readFileSync(fn + ".md", "utf8") if (fileExistsSync(fn + "/index.md")) return fs.readFileSync(fn + "/index.md", "utf8") return null } const targetMd = md ? md : tryRead(path.join(docs, pathname)) if (targetMd && !/^\s*#+\s+@extends/m.test(targetMd)) return targetMd const dirs = [ path.join(root, "/node_modules/pxt-core/common-docs"), ...getBundledPackagesDocs() ]; for (const d of dirs) { const template = tryRead(path.join(d, pathname)) if (template) return pxt.docs.augmentDocs(template, targetMd) } return undefined; } export function getBundledPackagesDocs(): string[] { const handledDirectories = {}; const outputDocFolders: string[] = []; for (const bundledDir of pxt.appTarget.bundleddirs || []) { getPackageDocs(bundledDir, outputDocFolders, handledDirectories); } return outputDocFolders; /** * This needs to produce a topologically sorted array of the docs of `dir` and any required packages, * such that any package listed as a dependency / additionalFilePath of another * package is added to `folders` before the one that requires it. */ function getPackageDocs(packageDir: string, folders: string[], resolvedDirs: pxt.Map) { if (resolvedDirs[packageDir]) return; resolvedDirs[packageDir] = true; const jsonDir = path.join(packageDir, "pxt.json"); const pxtjson = fs.existsSync(jsonDir) && (readJson(jsonDir) as pxt.PackageConfig); // before adding this package, include the docs of any package this one depends upon. if (pxtjson) { /** * include the package this extends from first; * that may have dependencies that overlap with this one or that will later be * overwritten by this one **/ if (pxtjson.additionalFilePath) { getPackageDocs(path.join(packageDir, pxtjson.additionalFilePath), folders, resolvedDirs); } if (pxtjson.dependencies) { Object.keys(pxtjson.dependencies).forEach(dep => { const parts = /^file:(.+)$/i.exec(pxtjson.dependencies[dep]); if (parts) { getPackageDocs(path.join(packageDir, parts[1]), folders, resolvedDirs); } }); } } const docsDir = path.join(packageDir, "docs"); if (fs.existsSync(docsDir)) { folders.push(docsDir); } } } export function lazyDependencies(): pxt.Map { // find pxt-core package const deps: pxt.Map = {}; [path.join("node_modules", "pxt-core", "package.json"), "package.json"] .filter(f => fs.existsSync(f)) .map(f => readJson(f)) .forEach(config => config && config.lazyDependencies && Util.jsonMergeFrom(deps, config.lazyDependencies)) return deps; } export function lazyRequire(name: string, install = false): any { /* tslint:disable:non-literal-require */ let r: any; try { r = require(name); } catch (e) { pxt.debug(e); pxt.debug((require.resolve).paths(name)); r = undefined; } if (!r && install) pxt.log(`package "${name}" failed to load, run "pxt npminstallnative" to install native depencencies`) return r; /* tslint:enable:non-literal-require */ } export function stringify(content: any) { if (process.env["PXT_ENV"] === "production") { return JSON.stringify(content); } return JSON.stringify(content, null, 4); } init();