/* tslint:disable:forin cli only run in node */

/// <reference path="../built/pxtlib.d.ts"/>

/// <reference path="../built/pxtcompiler.d.ts"/>

/// <reference path="../built/pxtpy.d.ts"/>

/// <reference path="../built/pxtsim.d.ts"/>

(global as any).pxt = pxt;

import * as nodeutil from './nodeutil';

import * as crypto from 'crypto';

import * as fs from 'fs';

import * as os from 'os';

import * as path from 'path';

import * as child_process from 'child_process';

import U = pxt.Util;

import Cloud = pxt.Cloud;

import Map = pxt.Map;

import * as server from './server';

import * as build from './buildengine';

import * as commandParser from './commandparser';

import * as hid from './hid';

import * as gdb from './gdb';

import * as clidbg from './clidbg';

import * as pyconv from './pyconv';

import * as gitfs from './gitfs';

import * as crowdin from './crowdin';

import * as youtube from './youtube';

const rimraf: (f: string, opts: any, cb: (err: any, res: any) => void) => void = require('rimraf');

pxt.docs.requireDOMSanitizer = () => require("sanitize-html");

let forceCloudBuild = process.env["KS_FORCE_CLOUD"] !== "no";

let forceLocalBuild = !!process.env["PXT_FORCE_LOCAL"];

let forceBuild = false; // don't use cache

Error.stackTraceLimit = 100;

function parseHwVariant(parsed: commandParser.ParsedCommand) {

let hwvariant = parsed && parsed.flags["hwvariant"] as string;

if (hwvariant) {

// map known variants

const knowVariants: pxt.Map<string> = {

"f4": "stm32f401",

"f401": "stm32f401",

"d5": "samd51",

"d51": "samd51",

"p0": "rpi",

"pi0": "rpi",

"pi": "rpi"

}

hwvariant = knowVariants[hwvariant.toLowerCase()] || hwvariant;

if (!/^hw---/.test(hwvariant)) hwvariant = 'hw---' + hwvariant;

}

return hwvariant;

}

function parseBuildInfo(parsed?: commandParser.ParsedCommand) {

const cloud = parsed && parsed.flags["cloudbuild"];

const local = parsed && parsed.flags["localbuild"];

const hwvariant = parseHwVariant(parsed);

forceBuild = parsed && !!parsed.flags["force"];

if (cloud && local)

U.userError("cannot specify local-build and cloud-build together");

if (cloud) {

forceCloudBuild = true;

forceLocalBuild = false;

}

if (local) {

forceCloudBuild = false;

forceLocalBuild = true;

}

if (hwvariant) {

pxt.log(`setting hardware variant to ${hwvariant}`);

pxt.setHwVariant(hwvariant)

}

}

const p = new commandParser.CommandParser();

function initTargetCommands() {

let cmdsjs = path.join(nodeutil.targetDir, 'built/cmds.js');

if (fs.existsSync(cmdsjs)) {

pxt.debug(`loading cli extensions...`)

let cli = require.main.require(cmdsjs)

if (cli.deployAsync) {

pxt.commands.deployFallbackAsync = cli.deployAsync

}

if (cli.addCommands) {

cli.addCommands(p)

}

}

}

let prevExports = (global as any).savedModuleExports

if (prevExports) {

module.exports = prevExports

}

export interface UserConfig {

localToken?: string;

noAutoBuild?: boolean;

noAutoStart?: boolean;

localBuild?: boolean;

noSerial?: boolean;

}

interface TargetPackageInfo {

options: pxtc.CompileOptions;

api: pxtc.ApisInfo;

sha: string;

}

let reportDiagnostic = reportDiagnosticSimply;

const targetJsPrefix = "var pxtTargetBundle = "

function reportDiagnostics(diagnostics: pxtc.KsDiagnostic[]): void {

for (const diagnostic of diagnostics) {

reportDiagnostic(diagnostic);

}

}

function reportDiagnosticSimply(diagnostic: pxtc.KsDiagnostic): void {

let output = pxtc.getDiagnosticString(diagnostic)

pxt.log(output);

}

function fatal(msg: string): Promise<any> {

pxt.log("Fatal error: " + msg)

throw new Error(msg)

}

export let globalConfig: UserConfig = {}

function homePxtDir() {

return path.join(process.env["HOME"] || process.env["UserProfile"], ".pxt")

}

function cacheDir() {

return path.join(homePxtDir(), "cache")

}

function configPath() {

return path.join(homePxtDir(), "config.json")

}

let homeDirsMade = false

function mkHomeDirs() {

if (homeDirsMade) return

homeDirsMade = true

if (!fs.existsSync(homePxtDir()))

fs.mkdirSync(homePxtDir())

if (!fs.existsSync(cacheDir()))

fs.mkdirSync(cacheDir())

}

function saveConfig() {

mkHomeDirs()

nodeutil.writeFileSync(configPath(), JSON.stringify(globalConfig, null, 4) + "\n")

}

function initConfigAsync(): Promise<void> {

let p = Promise.resolve();

let atok: string = process.env["PXT_ACCESS_TOKEN"]

if (fs.existsSync(configPath())) {

let config = <UserConfig>readJson(configPath())

globalConfig = config;

}

p.then(() => {

if (atok) {

let mm = /^(https?:.*)\?access_token=([\w\.]+)/.exec(atok)

if (!mm) {

console.error("Invalid accessToken format, expecting something like 'https://example.com/?access_token=0abcd.XXXX'")

return

}

Cloud.apiRoot = mm[1].replace(/\/$/, "").replace(/\/api$/, "") + "/api/"

Cloud.accessToken = mm[2]

}

});

return p;

}

function loadGithubToken() {

if (!pxt.github.token)

pxt.github.token = process.env["GITHUB_ACCESS_TOKEN"] || process.env["GITHUB_TOKEN"];

}

function searchAsync(...query: string[]) {

return pxt.packagesConfigAsync()

.then(config => pxt.github.searchAsync(query.join(" "), config))

.then(res => {

for (let r of res) {

console.log(`${r.fullName}: ${r.description}`)

}

})

}

function pkginfoAsync(repopath: string) {

let parsed = pxt.github.parseRepoId(repopath)

if (!parsed) {

console.log('Unknown repo');

return Promise.resolve();

}

const pkgInfo = (cfg: pxt.PackageConfig, tag?: string) => {

pxt.log(`name: ${cfg.name}`)

pxt.log(`description: ${cfg.description}`)

if (pxt.appTarget.appTheme)

pxt.log(`shareable url: ${pxt.appTarget.appTheme.embedUrl}#pub:gh/${parsed.fullName}${tag ? "#" + tag : ""}`)

}

return pxt.packagesConfigAsync()

.then(config => {

const status = pxt.github.repoStatus(parsed, config);

pxt.log(`github org: ${parsed.owner}`);

if (parsed.tag) pxt.log(`github tag: ${parsed.tag}`);

pxt.log(`package status: ${status == pxt.github.GitRepoStatus.Approved ? "approved" : status == pxt.github.GitRepoStatus.Banned ? "banned" : "neutral"}`)

if (parsed.tag)

return pxt.github.downloadPackageAsync(repopath, config)

.then(pkg => {

let cfg: pxt.PackageConfig = JSON.parse(pkg.files[pxt.CONFIG_NAME])

pkgInfo(cfg, parsed.tag)

pxt.debug(`size: ${JSON.stringify(pkg.files).length}`)

})

return pxt.github.pkgConfigAsync(parsed.fullName)

.then(cfg => {

pkgInfo(cfg)

return pxt.github.listRefsAsync(repopath)

.then(tags => {

pxt.log("tags: " + tags.join(", "))

return pxt.github.listRefsAsync(repopath, "heads")

})

.then(heads => {

pxt.log("branches: " + heads.join(", "))

})

})

})

}

export function pokeRepoAsync(parsed: commandParser.ParsedCommand): Promise<void> {

const repo = parsed.args[0];

let data = {

repo: repo,

getkey: false

}

if (parsed.flags["u"]) data.getkey = true

return Cloud.privatePostAsync("pokerepo", data)

.then(resp => {

console.log(resp)

})

}

export function apiAsync(path: string, postArguments?: string): Promise<void> {

if (postArguments == "delete") {

return Cloud.privateDeleteAsync(path)

.then(resp => console.log(resp))

}

if (postArguments == "-") {

return nodeutil.readResAsync(process.stdin)

.then(buf => buf.toString("utf8"))

.then(str => apiAsync(path, str))

}

if (postArguments && fs.existsSync(postArguments))

postArguments = fs.readFileSync(postArguments, "utf8");

let dat = postArguments ? JSON.parse(postArguments) : null

if (dat)

console.log("POST", "/api/" + path, JSON.stringify(dat, null, 2))

return Cloud.privateRequestAsync({

url: path,

data: dat

})

.then(resp => {

if (resp.json)

console.log(JSON.stringify(resp.json, null, 2))

else console.log(resp.text)

})

}

function uploadFileAsync(parsed: commandParser.ParsedCommand) {

const path = parsed.args[0];

let buf = fs.readFileSync(path)

let mime = U.getMime(path)

console.log("Upload", path)

return Cloud.privatePostAsync("upload/files", {

filename: path,

encoding: "base64",

content: buf.toString("base64"),

contentType: mime

})

.then(resp => {

console.log(resp)

})

}

let readlineCount = 0

function readlineAsync() {

process.stdin.resume();

process.stdin.setEncoding('utf8');

readlineCount++

return new Promise<string>((resolve, reject) => {

process.stdin.once('data', (text: string) => {

resolve(text)

})

})

}

export function queryAsync(msg: string, defl: string) {

process.stdout.write(`${msg} [${defl}]: `)

return readlineAsync()

.then(text => {

text = text.trim()

if (!text) return defl

else return text

})

}

export function yesNoAsync(msg: string): Promise<boolean> {

process.stdout.write(msg + " (y/n): ")

return readlineAsync()

.then(text => {

if (text.trim().toLowerCase() == "y")

return Promise.resolve(true)

else if (text.trim().toLowerCase() == "n")

return Promise.resolve(false)

else return yesNoAsync(msg)

})

}

function onlyExts(files: string[], exts: string[]) {

return files.filter(f => exts.indexOf(path.extname(f)) >= 0)

}

function pxtFileList(pref: string) {

return nodeutil.allFiles(pref + "webapp/public")

.concat(onlyExts(nodeutil.allFiles(pref + "built/web", 1), [".js", ".css"]))

.concat(nodeutil.allFiles(pref + "built/web/fonts", 1))

.concat(nodeutil.allFiles(pref + "built/web/vs", 4))

}

function semverCmp(a: string, b: string) {

let parse = (s: string) => {

let v = s.split(/\./).map(parseInt)

return v[0] * 100000000 + v[1] * 10000 + v[2]

}

return parse(a) - parse(b)

}

function checkIfTaggedCommitAsync() {

let currentCommit: string;

return nodeutil.gitInfoAsync(["rev-parse", "HEAD"])

.then(info => {

currentCommit = info.trim();

return nodeutil.gitInfoAsync(["ls-remote", "--tags"], undefined, true)

})

.then(info => {

const tagCommits = info.split("\n")

.map(line => {

const match = /^([a-fA-F0-9]+)\s+refs\/tags\/v\d+\.\d+\.\d+\^\{\}$/.exec(line);

return match && match[1]

});

return tagCommits.some(t => t === currentCommit)

});

}

let readJson = nodeutil.readJson;

function ciAsync() {

forceCloudBuild = true;

const buildInfo = ciBuildInfo();

pxt.log(`ci build using ${buildInfo.ci}`);

if (!buildInfo.tag)

buildInfo.tag = "";

if (!buildInfo.branch)

buildInfo.branch = "local"

const { tag, branch, pullRequest } = buildInfo

const atok = process.env.NPM_ACCESS_TOKEN

const npmPublish = /^v\d+\.\d+\.\d+$/.exec(tag) && atok;

if (npmPublish) {

let npmrc = path.join(process.env.HOME, ".npmrc")

pxt.log(`setting up ${npmrc}`)

let cfg = "//registry.npmjs.org/:_authToken=" + atok + "\n"

fs.writeFileSync(npmrc, cfg)

}

const latest = branch == "master" ? "latest" : "git-" + branch

// upload locs on build on master

const masterOrReleaseBranchRx = /^(master|v\d+\.\d+\.\d+)$/;

const apiStringBranchRx = pxt.appTarget.uploadApiStringsBranchRx

? new RegExp(pxt.appTarget.uploadApiStringsBranchRx)

: masterOrReleaseBranchRx;

const uploadDocs = !pullRequest

&& !!pxt.appTarget.uploadDocs

&& masterOrReleaseBranchRx.test(branch);

const uploadApiStrings = !pullRequest

&& (!!pxt.appTarget.uploadDocs || pxt.appTarget.uploadApiStringsBranchRx)

&& apiStringBranchRx.test(branch);

pxt.log(`tag: ${tag}`);

pxt.log(`branch: ${branch}`);

pxt.log(`latest: ${latest}`);

pxt.log(`pull request: ${pullRequest}`);

pxt.log(`upload api strings: ${uploadApiStrings}`);

pxt.log(`upload docs: ${uploadDocs}`);

function npmPublishAsync() {

if (!npmPublish) return Promise.resolve();

return nodeutil.runNpmAsync("publish");

}

let pkg = readJson("package.json")

if (pkg["name"] == "pxt-core") {

pxt.log("pxt-core build");

return checkIfTaggedCommitAsync()

.then(isTaggedCommit => {

pxt.log(`is tagged commit: ${isTaggedCommit}`);

let p = npmPublishAsync();

if (branch === "master" && isTaggedCommit) {

if (uploadDocs)

p = p

.then(() => buildWebStringsAsync())

.then(() => crowdin.execCrowdinAsync("upload", "built/webstrings.json"));

if (uploadApiStrings)

p = p.then(() => crowdin.execCrowdinAsync("upload", "built/strings.json"))

if (uploadDocs || uploadApiStrings)

p = p.then(() => crowdin.internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs));

}

return p;

});

} else {

pxt.log("target build");

return internalBuildTargetAsync()

.then(() => internalCheckDocsAsync(true))

.then(() => blockTestsAsync())

.then(() => npmPublishAsync())

.then(() => {

if (!process.env["PXT_ACCESS_TOKEN"]) {

// pull request, don't try to upload target

pxt.log('no token, skipping upload')

return Promise.resolve();

}

const trg = readLocalPxTarget();

const label = `${trg.id}/${tag || latest}`;

pxt.log(`uploading target with label ${label}...`);

return uploadTargetAsync(label);

})

.then(() => {

pxt.log("target uploaded");

if (uploadDocs || uploadApiStrings) {

return crowdin.internalUploadTargetTranslationsAsync(uploadApiStrings, uploadDocs)

.then(() => pxt.log("translations uploaded"));

} else {

pxt.log("skipping translations upload");

return Promise.resolve();

}

});

}

}

function bumpPxtCoreDepAsync(): Promise<void> {

let pkg = readJson("package.json")

if (pkg["name"] == "pxt-core") return Promise.resolve(pkg)

let gitPull = Promise.resolve();

let commitMsg: string = "";

["pxt-core", "pxt-common-packages"].forEach(knownPackage => {

const modulePath = path.join("node_modules", knownPackage)

if (fs.existsSync(path.join(modulePath, ".git"))) {

gitPull = gitPull.then(() => nodeutil.spawnAsync({

cmd: "git",

args: ["pull"],

cwd: modulePath

}))

}

// not referenced

if (!fs.existsSync(path.join(modulePath, "package.json")))

return;

gitPull

.then(() => {

let kspkg = readJson(path.join(modulePath, "package.json"));

let currVer = pkg["dependencies"][knownPackage]

if (!currVer) return; // not referenced

let newVer = kspkg["version"]

if (currVer == newVer) {

console.log(`Referenced ${knownPackage} dep up to date: ${currVer}`)

return;

}

console.log(`Bumping ${knownPackage} dep version: ${currVer} -> ${newVer}`)

if (currVer != "*" && pxt.semver.strcmp(currVer, newVer) > 0) {

U.userError(`Trying to downgrade ${knownPackage}.`)

}

if (currVer != "*" && pxt.semver.majorCmp(currVer, newVer) < 0) {

U.userError(`Trying to automatically update major version, please edit package.json manually.`)

}

pkg["dependencies"][knownPackage] = newVer

nodeutil.writeFileSync("package.json", nodeutil.stringify(pkg) + "\n")

commitMsg += `${commitMsg ? ", " : ""}bump ${knownPackage} to ${newVer}`;

})

})

gitPull = gitPull

.then(() => commitMsg ? nodeutil.runGitAsync("commit", "-m", commitMsg, "--", "package.json") : Promise.resolve());

return gitPull;

}

function updateAsync() {

return Promise.resolve()

.then(() => nodeutil.runGitAsync("pull"))

.then(() => bumpPxtCoreDepAsync())

.then(() => nodeutil.runNpmAsync("install"));

}

function justBumpPkgAsync() {

ensurePkgDir()

return nodeutil.needsGitCleanAsync()

.then(() => mainPkg.loadAsync())

.then(() => {

let v = pxt.semver.parse(mainPkg.config.version)

v.patch++

return queryAsync("New version", pxt.semver.stringify(v))

})

.then(nv => {

let v = pxt.semver.parse(nv)

mainPkg.config.version = pxt.semver.stringify(v)

mainPkg.saveConfig()

})

.then(() => nodeutil.runGitAsync("commit", "-a", "-m", mainPkg.config.version))

.then(() => nodeutil.runGitAsync("tag", "v" + mainPkg.config.version))

}

function tagReleaseAsync(parsed: commandParser.ParsedCommand) {

const tag = parsed.args[0] as string;

const version = parsed.args[1] as string;

const npm = !!parsed.flags["npm"];

// check that ...-ref.json exists for that tag

const fn = path.join('docs', tag + "-ref.json");

pxt.log(`checking ${fn}`)

if (!fn)

U.userError(`file ${fn} does not exist`);

const v = pxt.semver.normalize(version);

const npmPkg = `pxt-${pxt.appTarget.id}`;

if (!pxt.appTarget.appTheme.githubUrl)

U.userError('pxtarget theme missing "githubUrl" entry');

// check that tag exists in github

pxt.log(`checking github ${pxt.appTarget.appTheme.githubUrl} tag v${v}`);

return U.requestAsync({

url: pxt.appTarget.appTheme.githubUrl.replace(/\/$/, '') + "/releases/tag/v" + v,

method: "GET"

})

// check that release exists in npm

.then(() => {

if (!npm) return Promise.resolve();

pxt.log(`checking npm ${npmPkg} release`)

return nodeutil.npmRegistryAsync(npmPkg)

.then(registry => {

// verify that npm version exists

if (!registry.versions[v])

U.userError(`cannot find npm package ${npmPkg}@${v}`);

const npmTag = tag == "index" ? "latest" : tag;

return nodeutil.runNpmAsync(`dist-tag`, `add`, `${npmPkg}@v${v}`, npmTag);

})

})

// all good update ref file

.then(() => {

// update index file

nodeutil.writeFileSync(fn, JSON.stringify({

"appref": "v" + v

}, null, 4))

// TODO commit changes

console.log(`please commit ${fn} changes`);

})

}

function bumpAsync(parsed?: commandParser.ParsedCommand) {

const bumpPxt = parsed && parsed.flags["update"];

const upload = parsed && parsed.flags["upload"];

if (fs.existsSync(pxt.CONFIG_NAME)) {

if (upload) throw U.userError("upload only supported on targets");

return Promise.resolve()

.then(() => nodeutil.runGitAsync("pull"))

.then(() => justBumpPkgAsync())

.then(() => nodeutil.runGitAsync("push", "--tags"))

.then(() => nodeutil.runGitAsync("push"))

}

else if (fs.existsSync("pxtarget.json"))

return Promise.resolve()

.then(() => nodeutil.runGitAsync("pull"))

.then(() => bumpPxt ? bumpPxtCoreDepAsync().then(() => nodeutil.runGitAsync("push")) : Promise.resolve())

.then(() => nodeutil.runNpmAsync("version", "patch"))

.then(() => nodeutil.runGitAsync("push", "--tags"))

.then(() => nodeutil.runGitAsync("push"))

.then(() => upload ? uploadTaggedTargetAsync() : Promise.resolve())

else {

throw U.userError("Couldn't find package or target JSON file; nothing to bump")

}

}

function uploadTaggedTargetAsync() {

forceCloudBuild = true

if (!pxt.github.token) {

fatal("GitHub token not found, please use 'pxt login' to login with your GitHub account to push releases.");

return Promise.resolve();

}

return nodeutil.needsGitCleanAsync()

.then(() => Promise.all([

nodeutil.currGitTagAsync(),

nodeutil.gitInfoAsync(["rev-parse", "--abbrev-ref", "HEAD"]),

nodeutil.gitInfoAsync(["rev-parse", "HEAD"])

]))

// only build target after getting all the info

.then(info =>

internalBuildTargetAsync()

.then(() => internalCheckDocsAsync(true))

.then(() => info))

.then(info => {

const repoSlug = "microsoft/pxt-" + pxt.appTarget.id

setCiBuildInfo(info[0], info[1], info[2], repoSlug)

process.env['PXT_RELEASE_REPO'] = "https://git:" + pxt.github.token + "@github.com/" + repoSlug + "-built"

let v = pkgVersion()

pxt.log("uploading " + v)

return uploadCoreAsync({

label: "v" + v,

fileList: pxtFileList("node_modules/pxt-core/").concat(targetFileList()),

pkgversion: v,

githubOnly: true,

fileContent: {}

})

})

}

function pkgVersion() {

let ver = readJson("package.json")["version"]

const info = ciBuildInfo()

if (!info.tag)

ver += "-" + (info.commit ? info.commit.slice(0, 6) : "local")

return ver

}

function targetFileList() {

let lst = onlyExts(nodeutil.allFiles("built"), [".js", ".css", ".json", ".webmanifest"])

.concat(nodeutil.allFiles(path.join(simDir(), "public")));

if (simDir() != "sim")

lst = lst.concat(nodeutil.allFiles(path.join("sim", "public"), 5, true))

pxt.debug(`target files (on disk): ${lst.join('\r\n ')}`)

return lst;

}

function uploadTargetAsync(label: string) {

return uploadCoreAsync({

label,

fileList: pxtFileList("node_modules/pxt-core/").concat(targetFileList()),

pkgversion: pkgVersion(),

fileContent: {}

})

}

export function uploadTargetReleaseAsync(parsed?: commandParser.ParsedCommand) {

parseBuildInfo(parsed);

const label = parsed.args[0];

const rebundle = !!parsed.flags["rebundle"];

return (rebundle ? rebundleAsync() : internalBuildTargetAsync())

.then(() => {

return uploadTargetAsync(label);

});

}

export function uploadTargetRefsAsync(repoPath: string) {

if (repoPath) process.chdir(repoPath);

return nodeutil.needsGitCleanAsync()

.then(() => Promise.all([

nodeutil.gitInfoAsync(["rev-parse", "HEAD"]),

nodeutil.gitInfoAsync(["config", "--get", "remote.origin.url"])

]))

.then(info => {

return gitfs.uploadRefs(info[0], info[1])

.then(() => {

return Promise.resolve();

});

})

}

interface UploadOptions {

fileList: string[];

pkgversion: string;

fileContent?: Map<string>;

label?: string

legacyLabel?: boolean;

target?: string;

localDir?: string;

githubOnly?: boolean;

builtPackaged?: string;

minify?: boolean;

noAppCache?: boolean;

}

interface BlobReq {

hash: string;

content: string;

encoding: string;

filename: string; // comment only

size: number; // ditto

}

type GitTree = Map<GitEntry>;

interface GitEntry {

hash?: string;

subtree?: GitTree;

}

interface CommitInfo {

tree: GitTree;

parents: string[];

message: string;

target: string;

}

function uploadFileName(p: string) {

// normalize /, \ before filtering

return p.replace(/\\/g, '\/')

.replace(/^.*(built\/web\/|\w+\/public\/|built\/)/, "")

}

function gitUploadAsync(opts: UploadOptions, uplReqs: Map<BlobReq>) {

let reqs = U.unique(U.values(uplReqs), r => r.hash)

console.log("Asking for", reqs.length, "hashes")

return Promise.resolve()

.then(() => Cloud.privatePostAsync("upload/status", {

hashes: reqs.map(r => r.hash)

}))

.then(resp => {

let missing = U.toDictionary(resp.missing as string[], s => s)

let missingReqs = reqs.filter(r => !!U.lookup(missing, r.hash))

let size = 0

for (let r of missingReqs) size += r.size

console.log("files missing: ", missingReqs.length, size, "bytes")

return Promise.map(missingReqs,

r => Cloud.privatePostAsync("upload/blob", r)

.then(() => {

console.log(r.filename + ": OK," + r.size + " " + r.hash)

}))

})

.then(() => {

let roottree: Map<GitEntry> = {}

let get = (tree: GitTree, path: string): GitEntry => {

let subt = U.lookup(tree, path)

if (!subt)

subt = tree[path] = {}

return subt

}

let lookup = (tree: GitTree, path: string): GitEntry => {

let m = /^([^\/]+)\/(.*)/.exec(path)

if (m) {

let subt = get(tree, m[1])

U.assert(!subt.hash)

if (!subt.subtree) subt.subtree = {}

return lookup(subt.subtree, m[2])

} else {

return get(tree, path)

}

}

for (let fn of Object.keys(uplReqs)) {

let e = lookup(roottree, fn)

e.hash = uplReqs[fn].hash

}

const info = ciBuildInfo()

let data: CommitInfo = {

message: "Upload from " + info.commitUrl,

parents: [],

target: pxt.appTarget.id,

tree: roottree,

}

console.log("Creating commit...")

return Cloud.privatePostAsync("upload/commit", data)

})

.then(res => {

console.log("Commit:", res)

return uploadToGitRepoAsync(opts, uplReqs)

})

}

function uploadToGitRepoAsync(opts: UploadOptions, uplReqs: Map<BlobReq>) {

let label = opts.label

if (!label) {

console.log('no label; skip release upload');

return Promise.resolve();

}

let tid = pxt.appTarget.id

if (U.startsWith(label, tid + "/"))

label = label.slice(tid.length + 1)

if (!/^v\d/.test(label)) {

console.log('label is not a version; skipping release upload');

return Promise.resolve();

}

let repoUrl = process.env["PXT_RELEASE_REPO"]

if (!repoUrl) {

console.log("no $PXT_RELEASE_REPO variable; not uploading label " + label)

return Promise.resolve()

}

nodeutil.mkdirP("tmp")

let trgPath = "tmp/releases"

let mm = /^https:\/\/([^:]+):([^@]+)@([^\/]+)(.*)/.exec(repoUrl)

if (!mm) {

U.userError("wrong format for $PXT_RELEASE_REPO")

}

console.log(`create release ${label} in ${repoUrl}`);

let user = mm[1]

let pass = mm[2]

let host = mm[3]

let netRcLine = `machine ${host} login ${user} password ${pass}\n`

repoUrl = `https://${user}@${host}${mm[4]}`

let homePath = process.env["HOME"] || process.env["UserProfile"]

let netRcPath = path.join(homePath, /^win/.test(process.platform) ? "_netrc" : ".netrc")

let prevNetRc = fs.existsSync(netRcPath) ? fs.readFileSync(netRcPath, "utf8") : null

let newNetRc = prevNetRc ? prevNetRc + "\n" + netRcLine : netRcLine

console.log("Adding credentials to " + netRcPath)

fs.writeFileSync(netRcPath, newNetRc, {

encoding: "utf8",

mode: '600'

})

let cuser = process.env["USER"] || ""

if (cuser && !/travis/.test(cuser))

user += "-" + cuser

const cred = [

"-c", "credential.helper=",

"-c", "user.name=" + user,

"-c", "user.email=" + user + "@build.pxt.io",

]

const gitAsync = (args: string[]) => nodeutil.spawnAsync({

cmd: "git",

cwd: trgPath,

args: cred.concat(args)

})

const info = ciBuildInfo()

return Promise.resolve()

.then(() => {

if (fs.existsSync(trgPath)) {

let cfg = fs.readFileSync(trgPath + "/.git/config", "utf8")

if (cfg.indexOf("url = " + repoUrl) > 0) {

return gitAsync(["pull", "--depth=3"])

} else {

throw U.userError(trgPath + " already exists; please remove it")

}

} else {

return nodeutil.spawnAsync({

cmd: "git",

args: cred.concat(["clone", "--depth", "3", repoUrl, trgPath]),

cwd: "."

})

}

})

.then(() => {

for (let u of U.values(uplReqs)) {

let fpath = path.join(trgPath, u.filename)

nodeutil.mkdirP(path.dirname(fpath))

fs.writeFileSync(fpath, u.content, { encoding: u.encoding })

}

// make sure there's always something to commit

fs.writeFileSync(trgPath + "/stamp.txt", new Date().toString())

})

.then(() => gitAsync(["add", "."]))

.then(() => gitAsync(["commit", "-m", "Release " + label + " from " + info.commitUrl]))

.then(() => gitAsync(["tag", label]))

.then(() => gitAsync(["push"]))

.then(() => gitAsync(["push", "--tags"]))

.then(() => {

})

.finally(() => {

if (prevNetRc == null) {

console.log("Removing " + netRcPath)

fs.unlinkSync(netRcPath)

} else {

console.log("Restoring " + netRcPath)

fs.writeFileSync(netRcPath, prevNetRc, {

mode: '600'

})

}

})

}

function uploadArtFile(fn: string): string {

if (!fn || /^(https?|data):/.test(fn)) return fn; // nothing to do

fn = fn.replace(/^\.?\/*/, "/")

return "@cdnUrl@/blob/" + gitHash(fs.readFileSync("docs" + fn)) + "" + fn;

}

function gitHash(buf: Buffer) {

let hash = crypto.createHash("sha1")

hash.update(Buffer.from("blob " + buf.length + "\u0000", "utf8"))

hash.update(buf)

return hash.digest("hex")

}

function uploadCoreAsync(opts: UploadOptions) {

let targetConfig = readLocalPxTarget();

let defaultLocale = targetConfig.appTheme.defaultLocale;

let hexCache = path.join("built", "hexcache");

let hexFiles: string[] = [];

if (fs.existsSync(hexCache)) {

hexFiles = fs.readdirSync(hexCache)

.filter(f => /\.hex$/.test(f))

.filter(f => fs.readFileSync(path.join(hexCache, f), { encoding: "utf8" }) != "SKIP")

.map((f) => `@cdnUrl@/compile/${f}`);

pxt.log(`hex cache:\n\t${hexFiles.join('\n\t')}`)

}

let logos = (targetConfig.appTheme as any as Map<string>);

let targetImages = Object.keys(logos)

.filter(k => /(logo|hero)$/i.test(k) && /^\.\//.test(logos[k]));

let targetImagesHashed = pxt.Util.unique(targetImages.map(k => uploadArtFile(logos[k])), url => url);

let targetEditorJs = "";

if (pxt.appTarget.appTheme && pxt.appTarget.appTheme.extendEditor)

targetEditorJs = "@commitCdnUrl@editor.js";

let targetFieldEditorsJs = "";

if (pxt.appTarget.appTheme && pxt.appTarget.appTheme.extendFieldEditors)

targetFieldEditorsJs = "@commitCdnUrl@fieldeditors.js";

let replacements: Map<string> = {

"/sim/simulator.html": "@simUrl@",

"/sim/siminstructions.html": "@partsUrl@",

"/sim/sim.webmanifest": "@relprefix@webmanifest",

"/embed.js": "@targetUrl@@relprefix@embed",

"/cdn/": "@commitCdnUrl@",

"/doccdn/": "@commitCdnUrl@",

"/sim/": "@commitCdnUrl@",

"/blb/": "@blobCdnUrl@",

"@timestamp@": "",

"data-manifest=\"\"": "@manifest@",

"var pxtConfig = null": "var pxtConfig = @cfg@",

"@defaultLocaleStrings@": defaultLocale ? "@commitCdnUrl@" + "locales/" + defaultLocale + "/strings.json" : "",

"@cachedHexFiles@": hexFiles.length ? hexFiles.join("\n") : "",

"@cachedHexFilesEncoded@": encodeURLs(hexFiles),

"@targetEditorJs@": targetEditorJs,

"@targetFieldEditorsJs@": targetFieldEditorsJs,

"@targetImages@": targetImagesHashed.length ? targetImagesHashed.join('\n') : '',

"@targetImagesEncoded@": targetImagesHashed.length ? encodeURLs(targetImagesHashed) : ""

}

if (opts.localDir) {

let cfg: pxt.WebConfig = {

"relprefix": opts.localDir,

"verprefix": "",

"workerjs": opts.localDir + "worker.js",

"monacoworkerjs": opts.localDir + "monacoworker.js",

"gifworkerjs": opts.localDir + "gifjs/gif.worker.js",

"serviceworkerjs": opts.localDir + "serviceworker.js",

"pxtVersion": pxtVersion(),

"pxtRelId": "",

"pxtCdnUrl": opts.localDir,

"commitCdnUrl": opts.localDir,

"blobCdnUrl": opts.localDir,

"cdnUrl": opts.localDir,

"targetVersion": opts.pkgversion,

"targetRelId": "",

"targetUrl": "",

"targetId": opts.target,

"simUrl": opts.localDir + "simulator.html",

"simserviceworkerUrl": opts.localDir + "simulatorserviceworker.js",

"simworkerconfigUrl": opts.localDir + "workerConfig.js",

"partsUrl": opts.localDir + "siminstructions.html",

"runUrl": opts.localDir + "run.html",

"docsUrl": opts.localDir + "docs.html",

"multiUrl": opts.localDir + "multi.html",

"isStatic": true,

}

const targetImagePaths = targetImages.map(k =>

`${opts.localDir}${path.join('./docs', logos[k])}`);

replacements = {