diff --git a/.env b/.env deleted file mode 100644 index 4209a5f..0000000 --- a/.env +++ /dev/null @@ -1,11 +0,0 @@ -DBURI=mongodb://127.0.0.1:27017/githubDatasource -GITHUB_CLIENT_ID=348bd7eb3decbaed020a -GITHUB_CLIENT_SECRET=56667d64224db4ebe480da4bc56c651b230cd722 -CONTEXT_URI=http://gitplugin.com:5001 -ONESELF_URI=http://localhost:5000 -SESSION_SECRET=G1tP1ug1n -DASHBOARD_URI=http://localhost:5000/dashboard -INTEGRATIONS_URI=http://localhost:5000/integrations -PORT=5001 -APP_ID=abc -APP_SECRET=123 diff --git a/.gitignore b/.gitignore index 5d46e4e..31b28c8 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,8 @@ node_modules # Users Environment Variables .lock-wscript -.idea \ No newline at end of file +.idea +.env + +#mac file system +.DS_Store diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000..2cb788b --- /dev/null +++ b/.jshintrc @@ -0,0 +1,15 @@ +{ + "boss": true, + "curly": true, + "eqeqeq": true, + "eqnull": true, + "immed": true, + "latedef": true, + "mocha" : true, + "newcap": true, + "noarg": true, + "node": true, + "sub": true, + "undef": true, + "unused": true +} diff --git a/app.js b/app.js index 9eaa643..c55cbc4 100644 --- a/app.js +++ b/app.js @@ -1,5 +1,6 @@ -var express = require("express"); -var session = require("express-session"); +/*jslint node: true */ +var express = require('express'); +var session = require('express-session'); var path = require('path'); var swig = require('swig'); var q = require('q'); @@ -7,20 +8,89 @@ var logger = require('morgan'); var bodyParser = require('body-parser'); var cookieParser = require('cookie-parser'); var mongoClient = require('mongodb').MongoClient; +var _ = require('underscore'); -var GithubEvents = require("./routes/githubEvents"); +var GithubEvents = require('./routes/githubEvents'); var MongoRepository = require('./routes/mongoRepository'); -var GithubOAuth = require("./routes/githubOAuth"); -var QdService = require("./routes/qdService"); +var GithubOAuth = require('./routes/githubOAuth'); +var QdService = require('./routes/qdService'); var app = express(); + + +/*jslint nomen: true */ app.use(express.static(path.join(__dirname, 'public'))); +/*jslint nomen: false */ + app.use(logger()); app.use(cookieParser()); app.use(bodyParser.urlencoded({ extended: true })); + +var winston = require('winston'); +var filename = process.env.LOGGINGDIR ? path.join(process.env.LOGGINGDIR, 'github.log') : 'github.log'; +winston.info('logging debug messages to ' + filename); +winston.add(winston.transports.File, { filename: filename, level: 'debug', json: false }); +winston.error('Errors will be logged here'); +winston.warn('Warns will be logged here'); +winston.info('Info will be logged here'); +winston.verbose('Verbose will be logged here'); +winston.debug('Debug will be logged here'); +winston.silly('Silly will be logged here'); + +winston.info('DBURI=' + process.env.DBURI); +var mongoUri = process.env.DBURI; + +winston.info('PORT=' + process.env.PORT); +var port = process.env.PORT || 5001; + +winston.info('APIURI=' + process.env.APIURI); +_.each(process.env, function(value, key){ + winston.info(key + '=' + value); +}); + +winston.info('SESSION_SECRET=' + process.env.SESSION_SECRET.substring(0,2) + '...'); var sessionSecret = process.env.SESSION_SECRET; + +var logger = winston; + +logger.logInfo = function(username, message, object){ + if(object) { + logger.info(username + ': ' + message, object); + } + else { + logger.info(username + ': ' + message, []); + } +} + +logger.logDebug = function(username, message, object){ + if(object) { + logger.debug(username + ': ' + message, object); + } + else { + logger.debug(username + ': ' + message, []); + } +} + +logger.logError = function(username, message, object){ + if(object) { + logger.error(username + ': ' + message, object); + } + else { + logger.error(username + ': ' + message, []); + } +} + +logger.logSilly = function(username, message, object){ + if(object) { + logger.silly(username + ': ' + message, object); + } + else { + logger.silly(username + ': ' + message, []); + } +} + app.use(session({ secret: sessionSecret, cookie: { @@ -36,49 +106,101 @@ app.engine('html', swig.renderFile); app.set('views', __dirname + '/views'); app.set('view engine', 'html'); -var port = process.env.PORT || 5001; +var attachLogger = function(req, res, next){ + req.logger = winston; + next(); +}; +app.use(attachLogger); + var server = app.listen(port, function () { - console.log("Listening on " + port); + logger.logInfo('', 'Listening on ' + port); }); var qdService = new QdService(); -var mongoUri = process.env.DBURI; + var mongoRepository; var githubEvents; var githubOAuth; mongoClient.connect(mongoUri, function (err, databaseConnection) { if (err) { - console.error("Could not connect to Mongodb with URI : " + mongoUri); - console.error(err); + logger.logError('Could not connect to Mongodb with URI: ', [mongoUri, err]); process.exit(1); } else { - console.log("connected to mongo : ", mongoUri); + logger.logInfo('connected to mongo: ', mongoUri); mongoRepository = new MongoRepository(databaseConnection); githubEvents = new GithubEvents(mongoRepository, qdService); githubOAuth = new GithubOAuth(app, mongoRepository, qdService); + + githubEvents.setLogger(logger); } }); -app.get("/", function (req, res) { +app.get('/', function (req, res) { + //req.session.integrationUri = req.headers['x-forwarded-host']; + req.session.appUri = req.headers.referer === undefined ? '/' : req.headers.referer.split('/').slice(0,3).join('/'); + process.env.appUri = req.session.appUri; req.session.oneselfUsername = req.query.username; req.session.registrationToken = req.query.token; - res.render('index'); + req.session.redirectUri = req.query.redirect_uri; + logger.logInfo(req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); + res.redirect('/auth/github'); }); -app.post("/authSuccess", function (req, res) { - var githubUsername = req.query.username; - var streamInfo = { - streamid: req.query.streamid, - writeToken: req.headers.authorization, - lastSyncDate: req.query.latestSyncField - }; - mongoRepository.findByGithubUsername(githubUsername) - .then(function (user) { - var userInfo = { - githubUsername: githubUsername, - accessToken: user.accessToken - }; - return githubEvents.sendGithubEvents(userInfo, streamInfo); - }); +app.get('/reauth', function (req, res) { + if(req.query.username === undefined){ + res.render('reauthUnknownUser'); + return; + } + + //req.session.integrationUri = req.headers['x-forwarded-host']; + req.session.appUri = process.env.CONTEXT_URI; + req.session.redirect = '/reauth/complete'; + req.session.reauth = true; + req.session.username = req.query.username; + logger.logInfo(req.query.username, 'reauthing', [req.session.appUri, req.query.token]); + logger.logInfo(req.query.username, 'serving reauth page'); + res.render('reauth'); +}); + +app.get('/reauth/complete', function (req, res) { + //req.session.integrationUri = req.headers['x-forwarded-host']; + req.session.appUri = null; + req.session.redirect = null; + req.session.reauth = null; + req.session.username = null; + req.session.registrationToken = null; + logger.logInfo(req.query.username, 'reauthing complete, ', [req.session.appUri, req.query.token]); + logger.logDebug(req.query.username, 'serving reauth complete page'); + res.render('reauthcomplete'); +}); + +// This is the entry point after auth, to do a sync. It's also used +// as the callback url in the stream +app.get('/authSuccess', function (req, res) { + var githubUsername = req.query.username; + var streamInfo = { + streamid: req.query.streamid, + writeToken: req.headers.authorization, + lastSyncDate: req.query.latestSyncField + }; + logger.logInfo(req.query.username, 'syncing data, [stream id, write token]', [streamInfo.streamid.substring(0, 2), streamInfo.writeToken.substring(0, 2)]); + + mongoRepository.findByGithubUsername(githubUsername) + .then(function(user) { + logger.logDebug(user.githubUsername, 'found in the database'); + var userInfo = { + githubUsername: githubUsername, + accessToken: user.accessToken, + displayName: user.displayName, + email: user.email + }; + + return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.APIURI); + }) + .catch(function(error){ + logger.logError(githubUsername, 'error while trying to sync', error); + }); + + res.status(200).send('ok, sync request acknowledged'); } ); \ No newline at end of file diff --git a/package.json b/package.json index bef2fb0..df040ac 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "cookie-parser": "~1.3.2", "express": "4.8.5", "express-session": "1.7.6", + "moment": "~2.8.1", "mongodb": "1.4.8", "morgan": "~1.2.3", "octonode": "~0.6.4", @@ -13,10 +14,9 @@ "passport-github": "~0.1.5", "q": "1.0.1", "request": "2.40.0", - "socket.io": "^1.1.0", "swig": "~1.4.2", - "moment": "~2.8.1", - "underscore": "^1.7.0" + "underscore": "^1.7.0", + "winston": "^1.0.0" }, "engines": { "node": "0.10.x", diff --git a/public/js/error.js b/public/js/error.js index 0155710..b32d8dc 100644 --- a/public/js/error.js +++ b/public/js/error.js @@ -1,5 +1,5 @@ $(document).ready(function(){ setTimeout(function () { - window.location.href = "http://localhost:5000/integrations"; + window.location.href = "http://app-staging.1self.co/integrations"; }, 2000); -}); \ No newline at end of file +}); diff --git a/routes/.eslintrc b/routes/.eslintrc new file mode 100644 index 0000000..bb3c020 --- /dev/null +++ b/routes/.eslintrc @@ -0,0 +1,5 @@ +{ + "env": { + "node": true + } +} diff --git a/routes/githubEvents.js b/routes/githubEvents.js index ed5bd90..9d028bc 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -1,21 +1,65 @@ -var _ = require("underscore"); +var _ = require('underscore'); var github = require('octonode'); var moment = require('moment'); +var request = require('request'); var Q = require('q'); +var path = require('path'); module.exports = function (mongoRepository, qdService) { + + var logger; + var self = this; + + this.checkToken = function(userInfo, requestFunction, localLogger){ + var deferred = Q.defer(); + var options = { + url: "https://api.github.com/user/emails?access_token=" + userInfo.accessToken, + headers: { + "User-Agent": "1self" + } + }; + + requestFunction(options, function (err, res) { + if(err || res === undefined || res.statusCode !== 200){ + localLogger.logError(userInfo.githubUsername, JSON.stringify(res)); + localLogger.logError(userInfo.githubUsername, 'token is invalid'); + var result = { + message: err, + code: 500 + }; + + // it's likely when a sync error is seen that the github api is preventing token attacks + if(res.statusCode === 401){ + result.code = 401; + } + + localLogger.logError(userInfo.githubUsername, result); + deferred.reject(result); + } + else{ + localLogger.logInfo(userInfo.githubUsername, 'token is valid'); + deferred.resolve(res); + } + }); + return deferred.promise; + }; + + + var getGithubPushEventsPerPage = function (page, userInfo) { + var githubUsername = userInfo.githubUsername; - var user_api_url = "/users/" + githubUsername; + var user_api_url = '/users/' + githubUsername; var client = github.client(userInfo.accessToken); - client.get(user_api_url, {}, function (err, status, body, headers) { + logger.logDebug(userInfo.githubUsername, 'fetching page [page, user_api_url, accessToken]', [page, user_api_url, userInfo.accessToken.substring(0,2)]); + client.get(user_api_url, {}, function () { }); var githubUser = client.user(githubUsername); var deferred = Q.defer(); githubUser.events(page, ['PushEvent'], function (err, pushEvents) { if (err) { - console.log("err " + err); + logger.logError(githubUsername, 'err ' + err); deferred.reject(err); } else { deferred.resolve(pushEvents); @@ -23,7 +67,8 @@ module.exports = function (mongoRepository, qdService) { }); return deferred.promise; }; - var fetchGithubPushEvents = function (userInfo, streamInfo) { + + var fetchGithubPushEvents = function (userInfo) { var deferred = Q.defer(); var pages = _.range(1, 11); var promiseArray = _.map(pages, function (page) { @@ -36,90 +81,307 @@ module.exports = function (mongoRepository, qdService) { }); return deferred.promise; }; - var filterEventsToBeSent = function (events, lastSyncDate) { + + var filterEventsToBeSent = function (userInfo, events, lastSyncDate) { + logger.logDebug(userInfo.githubUsername, 'filtering events, [event count, lastSyncDate]', [events.length, lastSyncDate]); var eventsToBeSent = function (event) { return moment(event.created_at).isAfter(lastSyncDate); }; - return _.filter(events, eventsToBeSent); + var result = _.filter(events, eventsToBeSent); + logger.logDebug(userInfo.githubUsername, 'events filtered, [event count]', [events.length]); + return result; }; - var convertEventsTo1SelfFormat = function (filteredEvents) { - var convertEventTo1SelfFormat = function (event) { - var clone = function (obj) { - return JSON.parse(JSON.stringify(obj)); - }; - var singleEventTemplate = { - "actionTags": [ - "Github", - "Push" - ], - "source": "GitHub", - "objectTags": [ - "Computer", - "Software", - "Source Control" - ], - "dateTime": moment(event.created_at).toISOString(), - "latestSyncField": { - "$date": moment(event.created_at).toISOString() - }, - "properties": {} - }; - return clone(singleEventTemplate); + + var convertEventsTo1SelfFormat = function (filteredEvents, username) { + + var convertEventTo1SelfFormat = function (acc, event) { + logger.logSilly(username, 'converting raw event: ', event); + if (event.type === 'PushEvent') { + var singleEventTemplate = { + 'actionTags': [ + 'push' + ], + 'objectTags': [ + 'git', + 'github', + 'computer', + 'software', + 'source-control' + ], + 'id': event.payload.push_id, + 'childIds': _.map(event.payload.commits, function (c) { + return c.sha; + }), + 'dateTime': moment(event.created_at).toISOString(), + 'latestSyncField': { + '$date': moment(event.created_at).toISOString() + }, + 'properties': { + 'commits': event.payload.size, + 'repo': event.repo.name + } + }; + + logger.logSilly(username, 'converted push event: ', singleEventTemplate); + return acc.concat(singleEventTemplate); + + } + else if (event.commit !== undefined) { + var extensionStats = _.reduce(event.files, function(result, file){ + var ext = path.extname(file.filename).substring(1); + result[ext] = result[ext] || {}; + result[ext]['line-additions'] = (result[ext]['line-additions'] || 0) + file.additions; + result[ext]['line-deletions'] = (result[ext]['line-deletions'] || 0) + file.deletions; + result[ext]['line-changes'] = (result[ext]['line-changes'] || 0) + file.changes; + return result; + }, {}); + var commitEventTemplate = { + 'actionTags': [ + 'commit' + ], + 'objectTags': [ + 'git', + 'github', + 'computer', + 'software', + 'source-control' + ], + 'dateTime': moment(event.commit.author.date).toISOString(), + 'latestSyncField': { + '$date': moment(event.commit.author.date).toISOString() + }, + 'id': event.sha, + 'parentId': event.pushId, + 'url': event.commit.url, + 'info': { + 'message': event.commit.message, + 'author-name': event.commit.author.name, + 'author-email': event.commit.author.email, + 'author-date': event.commit.author.date + }, + 'properties': { + 'line-changes': event.stats.total, + 'line-additions': event.stats.additions, + 'line-deletions': event.stats.deletions, + 'file-changes': event.files.length, + 'repo': event.repo, + 'file-types': extensionStats + } + }; + + if(event.committerIsAuthor === false){ + commitEventTemplate.actionTags = ['merge']; + } + + if (event.commit.author.email !== event.commit.committer.email) { + commitEventTemplate.actionTags = ['patch']; + } + + logger.logSilly(username, 'converted commit event: ', commitEventTemplate); + + return acc.concat(commitEventTemplate); + } + else { + logger.logSilly(username, 'couldnt convert as event is not a push or commit'); + return acc; + } }; - return _.map(filteredEvents, convertEventTo1SelfFormat); + var mappedEvents = _.reduce(filteredEvents, convertEventTo1SelfFormat, []); + logger.logDebug(username, 'mapped events to 1self format, [event count]', mappedEvents.length); + + return mappedEvents; }; - var sendEventsToQD = function (events, streamInfo) { + + var sendEventsToQD = function (events, streamInfo, appUri, userInfo) { var deferred = Q.defer(); if (_.isEmpty(events)) { + logger.logDebug(userInfo.githubUsername, 'there are no events to send'); deferred.resolve(); + return; } - qdService.sendBatchEvents(events, streamInfo) + + logger.logDebug(userInfo.githubUsername, 'sending events to qd [event count, app uri]', [events.length, appUri]); + qdService.sendBatchEvents(events, streamInfo, appUri) .then(function () { - console.log("Events sent to 1self!!!"); + logger.logDebug(userInfo.githubUsername, 'events sent to 1self, [event count, stream info]', [events.length, streamInfo]); deferred.resolve(); }, function (error) { + logger.logError(userInfo.githubUsername, 'error while sending events', error); deferred.reject(error); }); return deferred.promise; }; + var createSyncStartEvent = function () { return { - "dateTime": moment().toISOString(), - "objectTags": ["sync"], - "actionTags": ["start"], - "properties": { - "source": "GitHub" + 'dateTime': moment().toISOString(), + 'objectTags': ['1self', 'integration', 'sync'], + 'actionTags': ['start'], + 'properties': {} + }; + }; + + var createSyncErrorEvent = function (code) { + return { + 'dateTime': moment().toISOString(), + 'objectTags': ['1self', 'integration', 'sync'], + 'actionTags': ['error'], + 'properties': { + code: code, + reauth: process.env.GITHUB_INT_CONTEXT_URI + '/reauth' } }; }; + var createSyncCompleteEvent = function () { return { - "dateTime": moment().toISOString(), - "objectTags": ["sync"], - "actionTags": ["complete"], - "properties": { - "source": "GitHub" - } + 'dateTime': moment().toISOString(), + 'objectTags': ['1self', 'integration', 'sync'], + 'actionTags': ['complete'], + 'properties': {} }; }; - this.sendGithubEvents = function (userInfo, streamInfo) { + var getGithubCommitEvents = function (filteredEvents, userInfo) { + logger.logDebug(userInfo.githubUsername, 'getting commit events for push events, [event count]', filteredEvents.length); + var deferred = Q.defer(); + + var commitObjects = []; + + var userEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(userInfo.email); + var userEmail = userEmailParts ? userEmailParts[1] + userEmailParts[3] : null; + + logger.logDebug(userInfo.githubUsername, 'flattening commits'); + logger.logDebug(userInfo.githubUsername, 'removing push email alias, [original, unaliased]', [userInfo.email, userEmail]); + + + _.each(filteredEvents, function (event) { + _.each(event.payload.commits, function (commit) { + var commitReq = { + url: commit.url, + pushId: event.payload.push_id, + repo: event.repo.name + }; + + commitReq.committerIsAuthor = true; + + if(userEmail){ + var commitEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(commit.author.email); + if(commitEmailParts && commitEmailParts.length === 4){ + var commitEmail = commitEmailParts[1] + commitEmailParts[3]; + commitReq.committerIsAuthor = commitEmail === userEmail; + } + } + + logger.logDebug(userInfo.githubUsername, 'commit req', commitReq); + commitObjects.push(commitReq); + }); + }); + + logger.logDebug(userInfo.githubUsername, 'commit have been flattened, [commits]', commitObjects); + + // the delay is in there as the github api is returning an error when hit with lots of requests + // quickly. + var getCommitPromise = function (commitObject, delay) { + var deferred = Q.defer(); + var url = commitObject.url + '?access_token=' + userInfo.accessToken; + var options = { + url: url, + headers: { + 'User-Agent': '1self' + } + }; + + setTimeout(function(){ + logger.logDebug(userInfo.githubUsername, [options.url, 'requesting commit, [options, delay]'].join(': '), [options, delay]); + request(options, function (err, res, body) { + if (!err) { + logger.logDebug(userInfo.githubUsername, [options.url, 'request successful'].join(': ')); + var commit = JSON.parse(body); + commit.pushId = commitObject.pushId; + commit.repo = commitObject.repo; + commit.committerIsAuthor = commitObject.committerIsAuthor; + deferred.resolve(commit); + } + else { + logger.logDebug(userInfo.githubUsername, [options.url, 'request failed, [error]'].join(': '), err); + deferred.reject(err); + } + }).end(); + }, + delay); + return deferred.promise; + }; + + var promiseArray = []; + + _.map(commitObjects, function (commitObject, i) { + promiseArray.push(getCommitPromise(commitObject, i * 100)); + }); + + Q.all(promiseArray).then(function (commitEvents) { + logger.logDebug(userInfo.githubUsername, 'all commit events retrieved'); + var events = filteredEvents.concat(commitEvents); + deferred.resolve(events); + }).catch(function (error) { + logger.logError(userInfo.githubUsername, 'Error occurred :: getGithubCommitEvents', error); + deferred.reject(error); + }); + + return deferred.promise; + }; + + this.sendGithubEvents = function (userInfo, streamInfo, appUri) { + logger.logInfo(userInfo.githubUsername, 'starting sync, sending start sync event, [appUri]', [appUri]); var syncStartEvent = createSyncStartEvent(); - qdService.sendEvent(syncStartEvent, streamInfo) + + qdService.sendEvent(syncStartEvent, streamInfo, appUri) + .then(function(){ + logger.logInfo(userInfo.githubUsername, 'testing token validity'); + logger.logInfo(userInfo.githubUsername, userInfo); + return self.checkToken(userInfo, request, logger); + }) .then(function () { - return fetchGithubPushEvents(userInfo, streamInfo) + logger.logInfo(userInfo.githubUsername, 'fetching events from github api', []); + return fetchGithubPushEvents(userInfo); }) .then(function (events) { - return filterEventsToBeSent(events, streamInfo.lastSyncDate); + logger.logInfo(userInfo.githubUsername, 'filtering events to include only pushes, [event count]', [events.length]); + return filterEventsToBeSent(userInfo, events, streamInfo.lastSyncDate); + }) + .then(function (filteredEvents) { + logger.logInfo(userInfo.githubUsername, 'getting the commit events for the pushes, [event count]', filteredEvents.length); + return getGithubCommitEvents(filteredEvents, userInfo); + }) + .then(function (filteredEvents){ + logger.logInfo(userInfo.githubUsername, 'converting events to 1self format, [event count]', filteredEvents.length); + return convertEventsTo1SelfFormat(filteredEvents, userInfo.githubUsername); }) - .then(convertEventsTo1SelfFormat) .then(function (eventsToBeSent) { - return sendEventsToQD(eventsToBeSent, streamInfo); + logger.logInfo(userInfo.githubUsername, 'sending the events to 1self, [event count]', eventsToBeSent.length); + return sendEventsToQD(eventsToBeSent, streamInfo, appUri, userInfo); }) .then(function () { + logger.logInfo(userInfo.githubUsername, 'sending sync complete to 1self', []); var syncCompleteEvent = createSyncCompleteEvent(); - return qdService.sendEvent(syncCompleteEvent, streamInfo); - }); + return qdService.sendEvent(syncCompleteEvent, streamInfo, appUri); + }) + .then(function() { + logger.logInfo(userInfo.githubUsername, 'finished sending events to 1self'); + }) + .catch(function (error) { + logger.logError(userInfo.githubUsername, 'Error occurred :: sendGithubEvents', error); + logger.logError(userInfo.githubUsername, 'sending error event'); + var errorEvent = createSyncErrorEvent(error.code); + return qdService.sendEvent(errorEvent, streamInfo, appUri); + }) + .done() ; + }; + + this.setLogger = function(newLogger){ + logger = newLogger; }; + + }; diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 713fc3a..cd34609 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -2,58 +2,187 @@ var request = require("request"); var passport = require('passport'); var githubStrategy = require('passport-github').Strategy; var _ = require('underscore'); +var q = require('q'); +var logger = require('winston'); -var GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID; -var GITHUB_CLIENT_SECRET = process.env.GITHUB_CLIENT_SECRET; -var CONTEXT_URI = process.env.CONTEXT_URI; +var GITHUB_DATASOURCE_CLIENT_ID = process.env.GITHUB_DATASOURCE_CLIENT_ID; +var GITHUB_DATASOURCE_CLIENT_SECRET = process.env.GITHUB_DATASOURCE_CLIENT_SECRET; +var GITHUB_INT_CONTEXT_URI = process.env.GITHUB_INT_CONTEXT_URI; + +var logInfo = function(username, message, object){ + logger.info(username + ': ' + message, object); +} + +var logDebug = function(username, message, object){ + logger.debug(username + ': ' + message, object); +} + +var logError = function(username, message, object){ + logger.error(username + ': ' + message, object); +} + +logger.info('env'); +logger.info(GITHUB_DATASOURCE_CLIENT_ID); +logger.info(GITHUB_DATASOURCE_CLIENT_SECRET); +logger.info(GITHUB_INT_CONTEXT_URI); module.exports = function (app, mongoRepository, oneselfService) { - var handleGithubCallback = function (req, res) { + var handleNewAuthCallback = function(req, res){ var githubUser = req.user.profile; var githubUsername = githubUser.username; + req.session.accessToken = req.user.accessToken; req.session.githubUsername = githubUsername; var oneselfUsername = req.session.oneselfUsername; var registrationToken = req.session.registrationToken; - console.log("github User is : " + JSON.stringify(githubUser)); - var callbackUrl = CONTEXT_URI + '/authSuccess?username=' + githubUsername + logDebug('reading user and registration token out of the session'); + logDebug(githubUsername, "github User is: " + JSON.stringify(githubUser)); + var callbackUrl = GITHUB_INT_CONTEXT_URI + '/authSuccess?username=' + githubUsername + '&latestSyncField={{latestSyncField}}' + '&streamid={{streamid}}'; - var document = { + // we use the display name to spot commits that weren't made by the committer. + // this can happen when a rebase occurs. + var userInfo = { githubUsername: githubUsername, - accessToken: req.user.accessToken + accessToken: req.user.accessToken, + displayName: req.user.profile.displayName, + }; + + var getEmailAddress = function (accessToken) { + var deferred = q.defer(); + var options = { + url: "https://api.github.com/user/emails?access_token=" + accessToken, + headers: { + "User-Agent": "1self" + } + }; + request(options, function (err, res, body) { + if (!err) { + userInfo.email = JSON.parse(body)[0].email; + logDebug(userInfo.githubUsername, 'email retrieved from users github profile', userInfo.email); + deferred.resolve(); + } + else { + logDebug(userInfo.githubUsername, 'error trying to get email: ', err); + deferred.reject(err); + } + }); + return deferred.promise; }; var syncGithubEvents = function (callbackUrl, writeToken) { + request({ - method: 'POST', + method: 'GET', uri: callbackUrl, gzip: true, headers: { 'Authorization': writeToken } }, function (e, response, body) { + console.log("Started event sync"); }); }; - oneselfService.registerStream(oneselfUsername, registrationToken, callbackUrl) - .then(function (stream) { - mongoRepository.insert(document); - var callbackUrlForUser = callbackUrl - .replace('{{streamid}}', stream.streamid) - .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); - syncGithubEvents(callbackUrlForUser, stream.writeToken); - var redirectUrl = process.env.INTEGRATIONS_URI; - res.redirect(redirectUrl); - }, function (error) { - res.render('error', { - error: error + getEmailAddress(req.session.accessToken) + .then(function(){ + return mongoRepository.findByGithubUsername(userInfo.githubUsername); + }) + .then(function (user) { + // we hit this if we are doing a re-auth and the user already exists + oneselfService.registerStream(oneselfUsername, registrationToken, process.env.APIURI, callbackUrl) + .then(function (stream) { + mongoRepository.insert(userInfo) + .then(function () { + var callbackUrlForUser = callbackUrl + .replace('{{streamid}}', stream.streamid) + .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); + syncGithubEvents(callbackUrlForUser, stream.writeToken); + + var redirectUri = req.session.redirectUri + '?success=true'; + res.redirect(redirectUri); + }); + }, function (error) { + var redirectUri = req.session.redirectUri + '?success=false&error=server_error'; + res.redirect(redirectUri); }); - }).catch(function (error) { - console.error("Error in github callback: ", error); + }) + .catch(function (error) { + logError(userInfo.githubUsername, "Error in github callback: ", error); + }); + }; + + var handleReauthCallback = function(req, res){ + var githubUser = req.user.profile; + var githubUsername = githubUser.username; + + logDebug(githubUsername, "github User is: ", JSON.stringify(githubUser)); + var callbackUrl = GITHUB_INT_CONTEXT_URI + '/authSuccess?username=' + githubUsername + + '&latestSyncField={{latestSyncField}}' + + '&streamid={{streamid}}'; + + var userInfo = { + githubUsername: githubUsername, + accessToken: req.user.accessToken, + displayName: req.user.profile.displayName, + }; + + var getEmailAddress = function (accessToken) { + var deferred = q.defer(); + var options = { + url: "https://api.github.com/user/emails?access_token=" + accessToken, + headers: { + "User-Agent": "1self" + } + }; + request(options, function (err, res, body) { + if (!err) { + userInfo.email = JSON.parse(body)[0].email; + logDebug(userInfo.githubUsername, 'email retrieved from users github profile', userInfo.email); + deferred.resolve(); + } + else { + logDebug(userInfo.githubUsername, 'error trying to get email: ', err); + deferred.reject(err); + } }); + return deferred.promise; + }; + + getEmailAddress(userInfo.accessToken) + .then(function(){ + return mongoRepository.findByGithubUsername(githubUsername); + }) + .then(function (user) { + var query = { + _id: user._id + }; + + var operation = { + $set: { + accessToken: userInfo.accessToken, + email: userInfo.email + } + }; + + mongoRepository.update(query, operation) + .then(function () { + res.redirect(req.session.redirect); + }); + }) + .catch(function (error) { + logError(userInfo.githubUsername, "Error in github callback: ", error); + }); + }; + var handleGithubCallback = function (req, res) { + if(req.session.reauth){ + handleReauthCallback(req, res); + } + else{ + handleNewAuthCallback(req, res); + } }; passport.serializeUser(function (user, done) { @@ -64,10 +193,13 @@ module.exports = function (app, mongoRepository, oneselfService) { done(null, obj); }); + console.log(GITHUB_DATASOURCE_CLIENT_ID); + console.log(GITHUB_DATASOURCE_CLIENT_SECRET); + console.log(GITHUB_INT_CONTEXT_URI); passport.use(new githubStrategy({ - clientID: GITHUB_CLIENT_ID, - clientSecret: GITHUB_CLIENT_SECRET, - callbackURL: CONTEXT_URI + "/auth/github/callback" + clientID: GITHUB_DATASOURCE_CLIENT_ID, + clientSecret: GITHUB_DATASOURCE_CLIENT_SECRET, + callbackURL: GITHUB_INT_CONTEXT_URI + "/auth/github/callback" }, function (accessToken, refreshToken, profile, done) { var githubProfile = { @@ -81,11 +213,11 @@ module.exports = function (app, mongoRepository, oneselfService) { app.use(passport.session()); app.get('/auth/github', passport.authenticate('github', { - scope: 'repo' + scope: 'repo,user:email' })); app.get('/auth/github/callback', passport.authenticate('github', { - failureRedirect: CONTEXT_URI + failureRedirect: GITHUB_INT_CONTEXT_URI, }), handleGithubCallback); } ; diff --git a/routes/mongoRepository.js b/routes/mongoRepository.js index 93f0c7a..20d52e0 100644 --- a/routes/mongoRepository.js +++ b/routes/mongoRepository.js @@ -8,12 +8,14 @@ module.exports = function (mongoConnection) { console.log("connection object is : " + this.mongoConnection); }; + var userCollection = 'githubDatasourceUsers'; + this.findByGithubUsername = function (username) { var deferred = Q.defer(); var query = { githubUsername: username }; - mongoConnection.collection('users').findOne(query, function (error, user) { + mongoConnection.collection(userCollection).findOne(query, function (error, user) { if (error) { deferred.reject(error); } else { @@ -25,7 +27,7 @@ module.exports = function (mongoConnection) { this.insert = function (document) { var deferred = Q.defer(); - mongoConnection.collection('users').insert(document, function (error, numberOfDocsInserted) { + mongoConnection.collection(userCollection).insert(document, function (error, numberOfDocsInserted) { if (error) { deferred.reject(error); } else { @@ -37,12 +39,8 @@ module.exports = function (mongoConnection) { this.update = function (findQuery, updateQuery) { var deferred = Q.defer(); - mongoConnection.collection('users', function (err, collection) { - collection.update(findQuery, { - $set: updateQuery - }, { - upsert: true - }, function (error, data) { + mongoConnection.collection(userCollection, function (err, collection) { + collection.update(findQuery, updateQuery, function (error, data) { if (error) { deferred.reject(error); } else { diff --git a/routes/qdService.js b/routes/qdService.js index b1b23e7..1ba658b 100644 --- a/routes/qdService.js +++ b/routes/qdService.js @@ -1,18 +1,25 @@ var requestModule = require('request'); var Q = require('q'); +var logger = require('winston'); module.exports = function () { - var appId = process.env.APP_ID; - var appSecret = process.env.APP_SECRET; - var oneselfUri = process.env.ONESELF_URI; + var appId = process.env.GITHUB_APP_ID; + var appSecret = process.env.GITHUB_APP_SECRET; + var oneselfUri = process.env.CONTEXT_URI; - this.registerStream = function (oneselfUsername, token, callbackUrl) { + logger.info('1self service env'); + logger.info(process.env.GITHUB_APP_ID); + logger.info(process.env.GITHUB_APP_SECRET); + logger.info(process.env.CONTEXT_URI); + this.registerStream = function (oneselfUsername, token, appUri, callbackUrl) { var deferred = Q.defer(); - console.log("Registering stream..."); - + logger.debug("Registering stream...", oneselfUsername, token, callbackUrl, appUri); + + var requestUri = appUri + '/v1/users/' + oneselfUsername + '/streams'; + logger.debug(requestUri); var options = { method: 'POST', - uri: oneselfUri + '/v1/users/' + oneselfUsername + '/streams', + uri: appUri + '/v1/users/' + oneselfUsername + '/streams', headers: { 'Authorization': appId + ':' + appSecret, 'registration-token': token @@ -23,27 +30,33 @@ module.exports = function () { } }; requestModule(options, function (e, response, body) { + if (e) { + logger.error('error while registering stream: ', e); + deferred.reject("Error: ", e); + return; + } + if (response.statusCode === 401) { + logger.error('error while registering stream: ', [response.statusCode, response.body]); deferred.reject('auth error: check your appId and appSecret', null); return; } if (response.statusCode === 400) { + logger.error('error while registering stream: ', [response.statusCode, response.body]); deferred.reject('Invalid username and registrationToken', null); return; } - if (e) { - deferred.reject("Error: ", e); - } + deferred.resolve(body); }); return deferred.promise; }; - this.sendBatchEvents = function (events, streamInfo) { + this.sendBatchEvents = function (events, streamInfo, appUri) { var deferred = Q.defer(); var options = { method: 'POST', - uri: oneselfUri + '/v1/streams/' + streamInfo.streamid + '/events/batch', + uri: appUri + '/v1/streams/' + streamInfo.streamid + '/events/batch', gzip: true, headers: { 'Authorization': streamInfo.writeToken, @@ -64,11 +77,11 @@ module.exports = function () { return deferred.promise; }; - this.sendEvent = function (event, streamInfo) { + this.sendEvent = function (event, streamInfo, appUri) { var deferred = Q.defer(); var options = { method: 'POST', - uri: oneselfUri + '/v1/streams/' + streamInfo.streamid + '/events', + uri: appUri + '/v1/streams/' + streamInfo.streamid + '/events', gzip: true, headers: { 'Authorization': streamInfo.writeToken, @@ -88,4 +101,31 @@ module.exports = function () { }); return deferred.promise; }; + + this.link = function(oneselfUsername, streamId, appUri) { + var deferred = Q.defer(); + var options = { + method: 'POST', + uri: appUri + '/v1/users/' + oneselfUsername + '/link', + gzip: true, + headers: { + 'Content-type': 'application/json' + }, + json: true, + body: { + "streamId": streamId, + "appId" : appId + } + }; + requestModule(options, function (err, response, body) { + if (err) { + deferred.reject(err); + } + if (response.statusCode === 400) { + deferred.reject("Invalid streamId and appId") + } + deferred.resolve(); + }); + return deferred.promise; + } }; diff --git a/routes/test/githubEvents.tests.js b/routes/test/githubEvents.tests.js new file mode 100644 index 0000000..b44ddad --- /dev/null +++ b/routes/test/githubEvents.tests.js @@ -0,0 +1,102 @@ +'use strict'; + +var assert = require('assert'); +var GithubEvents = require('../githubEvents.js'); + +var events = new GithubEvents(null, null); + +var logger = { + messages:{ + verbose: [], + info: [], + warn: [], + debug: [], + silly: [] + }, + logDebug: function(message){ + this.messages.debug.push(message); + }, + logInfo: function(message){ + this.messages.info.push(message); + }, + logError: function(message){ + this.messages.info.push(message); + } +}; + +describe('githubEvents', function () { + it('check token rejects on any error', function () { + var userInfo = { + accessToken: 'at' + }; + + var request = function(options, callback){ + callback("error, unknown", {statusCode: 500}); + }; + + return events.checkToken(userInfo, request, logger) + .then(function(){ + assert(false, 'error wasnt rejected'); + }) + .catch(function(error){ + assert.equal(error.code, 500); + }); + }); + + it('check token translates 401 from github to 401 on our error', function () { + var userInfo = { + accessToken: 'at' + }; + + var request = function(options, callback){ + callback(null, {statusCode: 401}); + }; + + return events.checkToken(userInfo, request, logger) + .then(function(){ + assert(false, 'error wasnt rejected'); + }) + .catch(function(error){ + assert.equal(error.code, 401); + }); + }); + + it('check all other response errors are 500', function () { + var userInfo = { + accessToken: 'at' + }; + + var request = function(options, callback){ + // this response code doesn't mean anything, it's just not a 200 + callback(null, {statusCode: 480}); + }; + + return events.checkToken(userInfo, request, logger) + .then(function(){ + assert(false, 'error wasnt rejected'); + }) + .catch(function(error){ + assert.equal(error.code, 500); + }); + }); + + it('check 200 response gives a valid token', function () { + var userInfo = { + accessToken: 'at' + }; + + var request = function(options, callback){ + callback(null, {statusCode: 200}); + }; + + return events.checkToken(userInfo, request, logger) + .then(function(){ + assert(true, 'error wasnt rejected'); + }) + .catch(function(){ + assert(false, 'shouldnt get an error'); + }); + }); +}); + + diff --git a/views/reauth.html b/views/reauth.html new file mode 100644 index 0000000..be0bdb0 --- /dev/null +++ b/views/reauth.html @@ -0,0 +1,20 @@ + + + + + + + + 1self Github Integration + + +
+

Github Integration for 1self

+

Reauthorize the github plugin

+ Reauth +
+ + + + + \ No newline at end of file diff --git a/views/reauthUnknownUser.html b/views/reauthUnknownUser.html new file mode 100644 index 0000000..ba58860 --- /dev/null +++ b/views/reauthUnknownUser.html @@ -0,0 +1,18 @@ + + + + + + + + 1self Github Integration + + +
+

Reauth: unknown user

+
+ + + + + \ No newline at end of file diff --git a/views/reauthcomplete.html b/views/reauthcomplete.html new file mode 100644 index 0000000..212fc2b --- /dev/null +++ b/views/reauthcomplete.html @@ -0,0 +1,18 @@ + + + + + + + + 1self Github Integration + + +
+

Reauth successful

+
+ + + + + \ No newline at end of file