From fc05c33ce677621357414995a0b9d394f2f75465 Mon Sep 17 00:00:00 2001 From: Chinmay Naik Date: Thu, 26 Feb 2015 14:48:51 +0530 Subject: [PATCH 01/66] fix insertion of user record in database. --- .env | 2 +- routes/githubOAuth.js | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.env b/.env index 4209a5f..9f1b04b 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -DBURI=mongodb://127.0.0.1:27017/githubDatasource +DBURI=mongodb://127.0.0.1:27017/githubdatasource GITHUB_CLIENT_ID=348bd7eb3decbaed020a GITHUB_CLIENT_SECRET=56667d64224db4ebe480da4bc56c651b230cd722 CONTEXT_URI=http://gitplugin.com:5001 diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 713fc3a..3f94c9d 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -40,18 +40,21 @@ module.exports = function (app, mongoRepository, oneselfService) { oneselfService.registerStream(oneselfUsername, registrationToken, callbackUrl) .then(function (stream) { - mongoRepository.insert(document); - var callbackUrlForUser = callbackUrl - .replace('{{streamid}}', stream.streamid) - .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); - syncGithubEvents(callbackUrlForUser, stream.writeToken); - var redirectUrl = process.env.INTEGRATIONS_URI; - res.redirect(redirectUrl); - }, function (error) { - res.render('error', { - error: error - }); - }).catch(function (error) { + mongoRepository.insert(document) + .then(function () { + var callbackUrlForUser = callbackUrl + .replace('{{streamid}}', stream.streamid) + .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); + syncGithubEvents(callbackUrlForUser, stream.writeToken); + var redirectUrl = process.env.INTEGRATIONS_URI; + res.redirect(redirectUrl); + }, function (error) { + res.render('error', { + error: error + }); + }); + }) + .catch(function (error) { console.error("Error in github callback: ", error); }); }; From 882f8eb6f6fb8ac40a5e0e059d9e95c309c898a2 Mon Sep 17 00:00:00 2001 From: Devika Deshmukh Date: Fri, 27 Feb 2015 11:34:30 +0530 Subject: [PATCH 02/66] -- Fixed the redirect to error page bug --- routes/githubOAuth.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 3f94c9d..a5efdf7 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -48,11 +48,11 @@ module.exports = function (app, mongoRepository, oneselfService) { syncGithubEvents(callbackUrlForUser, stream.writeToken); var redirectUrl = process.env.INTEGRATIONS_URI; res.redirect(redirectUrl); - }, function (error) { - res.render('error', { - error: error - }); - }); + }) + }, function (error) { + res.render('error', { + error: error + }); }) .catch(function (error) { console.error("Error in github callback: ", error); From 371a1e7b9f35585929bbdbf7cc482f4d26a1ea06 Mon Sep 17 00:00:00 2001 From: Devika Deshmukh Date: Fri, 27 Feb 2015 17:19:38 +0530 Subject: [PATCH 03/66] -- Removed socket.io --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index bef2fb0..1712893 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,6 @@ "passport-github": "~0.1.5", "q": "1.0.1", "request": "2.40.0", - "socket.io": "^1.1.0", "swig": "~1.4.2", "moment": "~2.8.1", "underscore": "^1.7.0" From 2a7b5d49c2df8b2efd2259c9d6ec8fb25ff8fa6d Mon Sep 17 00:00:00 2001 From: Devika Deshmukh Date: Tue, 3 Mar 2015 12:35:09 +0530 Subject: [PATCH 04/66] -- Added migration code for backward compatibilty for existing github datasource users --- routes/githubOAuth.js | 63 +++++++++++++++++++++++++++++---------- routes/mongoRepository.js | 6 +--- routes/qdService.js | 26 ++++++++++++++++ 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index a5efdf7..418107a 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -37,22 +37,55 @@ module.exports = function (app, mongoRepository, oneselfService) { }, function (e, response, body) { }); }; + mongoRepository.findByGithubUsername(githubUsername) + .then(function (user) { + if (user && user.streamid) { + var callbackUrlForUser = callbackUrl + .replace('{{streamid}}', user.streamid) + .replace('{{latestSyncField}}', user.lastGithubSyncDate.toISOString()); - oneselfService.registerStream(oneselfUsername, registrationToken, callbackUrl) - .then(function (stream) { - mongoRepository.insert(document) - .then(function () { - var callbackUrlForUser = callbackUrl - .replace('{{streamid}}', stream.streamid) - .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); - syncGithubEvents(callbackUrlForUser, stream.writeToken); - var redirectUrl = process.env.INTEGRATIONS_URI; - res.redirect(redirectUrl); - }) - }, function (error) { - res.render('error', { - error: error - }); + syncGithubEvents(callbackUrlForUser, user.writeToken); + oneselfService.link(oneselfUsername, user.streamid) + .then(function () { + var findQuery = { + 'githubUsername': githubUsername + }; + var updateQuery = { + "$set": { + "accessToken": req.user.accessToken + }, + "$unset": { + "streamid": 1, + "readToken": 1, + "writeToken": 1, + "lastGithubSyncDate": 1 + } + }; + return mongoRepository.update(findQuery, updateQuery) + }) + .then(function () { + var redirectUrl = process.env.INTEGRATIONS_URI; + res.redirect(redirectUrl); + }); + } + else { + oneselfService.registerStream(oneselfUsername, registrationToken, callbackUrl) + .then(function (stream) { + mongoRepository.insert(document) + .then(function () { + var callbackUrlForUser = callbackUrl + .replace('{{streamid}}', stream.streamid) + .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); + syncGithubEvents(callbackUrlForUser, stream.writeToken); + var redirectUrl = process.env.INTEGRATIONS_URI; + res.redirect(redirectUrl); + }) + }, function (error) { + res.render('error', { + error: error + }); + }) + } }) .catch(function (error) { console.error("Error in github callback: ", error); diff --git a/routes/mongoRepository.js b/routes/mongoRepository.js index 93f0c7a..091f9a3 100644 --- a/routes/mongoRepository.js +++ b/routes/mongoRepository.js @@ -38,11 +38,7 @@ module.exports = function (mongoConnection) { this.update = function (findQuery, updateQuery) { var deferred = Q.defer(); mongoConnection.collection('users', function (err, collection) { - collection.update(findQuery, { - $set: updateQuery - }, { - upsert: true - }, function (error, data) { + collection.update(findQuery, updateQuery, function (error, data) { if (error) { deferred.reject(error); } else { diff --git a/routes/qdService.js b/routes/qdService.js index b1b23e7..5b00a44 100644 --- a/routes/qdService.js +++ b/routes/qdService.js @@ -88,4 +88,30 @@ module.exports = function () { }); return deferred.promise; }; + this.link = function(oneselfUsername, streamId) { + var deferred = Q.defer(); + var options = { + method: 'POST', + uri: oneselfUri + '/v1/users/' + oneselfUsername + '/link', + gzip: true, + headers: { + 'Content-type': 'application/json' + }, + json: true, + body: { + "streamId": streamId, + "appId" : appId + } + }; + requestModule(options, function (err, response, body) { + if (err) { + deferred.reject(err); + } + if (response.statusCode === 400) { + deferred.reject("Invalid streamId and appId") + } + deferred.resolve(); + }); + return deferred.promise; + } }; From 0107e5cbd663b47fb5003620c184d15161d0d475 Mon Sep 17 00:00:00 2001 From: Devika Deshmukh Date: Tue, 10 Mar 2015 10:48:33 +0530 Subject: [PATCH 05/66] -- Sync url now is a GET request --- app.js | 2 +- routes/githubOAuth.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app.js b/app.js index 9eaa643..73a1afa 100644 --- a/app.js +++ b/app.js @@ -65,7 +65,7 @@ app.get("/", function (req, res) { res.render('index'); }); -app.post("/authSuccess", function (req, res) { +app.get("/authSuccess", function (req, res) { var githubUsername = req.query.username; var streamInfo = { streamid: req.query.streamid, diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 418107a..9b97e20 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -28,13 +28,14 @@ module.exports = function (app, mongoRepository, oneselfService) { var syncGithubEvents = function (callbackUrl, writeToken) { request({ - method: 'POST', + method: 'GET', uri: callbackUrl, gzip: true, headers: { 'Authorization': writeToken } }, function (e, response, body) { + console.log("Synced github Events!!!"); }); }; mongoRepository.findByGithubUsername(githubUsername) @@ -43,7 +44,7 @@ module.exports = function (app, mongoRepository, oneselfService) { var callbackUrlForUser = callbackUrl .replace('{{streamid}}', user.streamid) .replace('{{latestSyncField}}', user.lastGithubSyncDate.toISOString()); - + console.log("Syncing github events"); syncGithubEvents(callbackUrlForUser, user.writeToken); oneselfService.link(oneselfUsername, user.streamid) .then(function () { From be562d91ed06f698c6329f85754df25f8058ccea Mon Sep 17 00:00:00 2001 From: Devika Deshmukh Date: Tue, 10 Mar 2015 11:35:09 +0530 Subject: [PATCH 06/66] send response for sync call --- app.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app.js b/app.js index 73a1afa..3c0d1e4 100644 --- a/app.js +++ b/app.js @@ -80,5 +80,6 @@ app.get("/authSuccess", function (req, res) { }; return githubEvents.sendGithubEvents(userInfo, streamInfo); }); + res.status(200).send("ok"); } ); \ No newline at end of file From e67652d470447cad8c60b1bf56fec23271e835d7 Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Mon, 13 Apr 2015 19:56:00 +0530 Subject: [PATCH 07/66] Delete .env --- .env | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 .env diff --git a/.env b/.env deleted file mode 100644 index 9f1b04b..0000000 --- a/.env +++ /dev/null @@ -1,11 +0,0 @@ -DBURI=mongodb://127.0.0.1:27017/githubdatasource -GITHUB_CLIENT_ID=348bd7eb3decbaed020a -GITHUB_CLIENT_SECRET=56667d64224db4ebe480da4bc56c651b230cd722 -CONTEXT_URI=http://gitplugin.com:5001 -ONESELF_URI=http://localhost:5000 -SESSION_SECRET=G1tP1ug1n -DASHBOARD_URI=http://localhost:5000/dashboard -INTEGRATIONS_URI=http://localhost:5000/integrations -PORT=5001 -APP_ID=abc -APP_SECRET=123 From aa3d37205b384315606c1d22c1ad932b788ebf5e Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Mon, 4 May 2015 16:51:30 +0530 Subject: [PATCH 08/66] Changes for storing gitHub commits on GitHub sync --- .gitignore | 3 +- routes/githubEvents.js | 138 +++++++++++++++++++++++++++++++++++------ 2 files changed, 121 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 5d46e4e..3f82a04 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,5 @@ node_modules # Users Environment Variables .lock-wscript -.idea \ No newline at end of file +.idea +.env diff --git a/routes/githubEvents.js b/routes/githubEvents.js index ed5bd90..067cac9 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -1,6 +1,7 @@ var _ = require("underscore"); var github = require('octonode'); var moment = require('moment'); +var request = require('request'); var Q = require('q'); module.exports = function (mongoRepository, qdService) { @@ -42,33 +43,76 @@ module.exports = function (mongoRepository, qdService) { }; return _.filter(events, eventsToBeSent); }; + var convertEventsTo1SelfFormat = function (filteredEvents) { var convertEventTo1SelfFormat = function (event) { var clone = function (obj) { return JSON.parse(JSON.stringify(obj)); }; - var singleEventTemplate = { - "actionTags": [ - "Github", - "Push" - ], - "source": "GitHub", - "objectTags": [ - "Computer", - "Software", - "Source Control" - ], - "dateTime": moment(event.created_at).toISOString(), - "latestSyncField": { - "$date": moment(event.created_at).toISOString() - }, - "properties": {} - }; + if (event.type == "PushEvent") { + var singleEventTemplate = { + "actionTags": [ + "Github", + "Push" + ], + "source": "GitHub", + "objectTags": [ + "Computer", + "Software", + "Source Control" + ], + "dateTime": moment(event.created_at).toISOString(), + "latestSyncField": { + "$date": moment(event.created_at).toISOString() + }, + "properties": { + "pushId": event.payload["push_id"], + "commitIds": _.map(event.payload.commits, function (c) { + return c['sha'] + }) + } + }; + } else if (event.commit != undefined) { + var singleEventTemplate = { + "actionTags": [ + "commit" + ], + "source": "GitHub", + "objectTags": [ + "git", + "github", + "computer", + "software", + "source control" + ], + "dateTime": moment(event.created_at).toISOString(), + "latestSyncField": { + "$date": moment(event.created_at).toISOString() + }, + "properties": { + "pushId": event.pushId, + "sha": event.sha, + "author-name": event.commit.author.name, + "author-email": event.commit.author.email, + "author-date": event.commit.author.date, + "message": event.commit.message, + "url": event.commit.url, + "line-changes": event.stats.total, + "line-additions": event.stats.additions, + "line-deletions": event.stats.deletions + //"file-changes": event.files.changes, + //"file-additions": event.files.additions, + //"file-deletions": event.files.deletions + } + }; + } else { + console.log("ERROR commit ---->", JSON.stringify(event)) + } return clone(singleEventTemplate); }; - return _.map(filteredEvents, convertEventTo1SelfFormat); }; + var sendEventsToQD = function (events, streamInfo) { var deferred = Q.defer(); if (_.isEmpty(events)) { @@ -104,8 +148,60 @@ module.exports = function (mongoRepository, qdService) { }; }; + var getGithubCommitEvents = function (filteredEvents) { + var deferred = Q.defer(); + + var commitObjects = []; + _.each(filteredEvents, function (event) { + _.each(event.payload.commits, function (commit) { + commitObjects.push({url: commit['url'], pushId: event.payload["push_id"]}) + }) + }); + + var getCommitPromise = function (commitObject) { + var deferred = Q.defer(); + console.log("Hitting request") + + var options = { + url: commitObject.url, + headers: { + "User-Agent": "1self" + } + }; + request(options, function (err, res, body) { + if (!err) { + var commit = JSON.parse(body); + commit.pushId = commitObject.pushId + deferred.resolve(commit); + } + else { + deferred.reject(err); + } + }); + return deferred.promise; + }; + + var promiseArray = []; + + //commitObjects = commitObjects.slice(0, 3); + + _.map(commitObjects, function (commitObject) { + promiseArray.push(getCommitPromise(commitObject)) + }); + + Q.all(promiseArray).then(function (commitEvents) { + var events = filteredEvents.concat(commitEvents) + deferred.resolve(events); + }).catch(function (error) { + console.log("Error occurred", error); + }); + + return deferred.promise; + }; + this.sendGithubEvents = function (userInfo, streamInfo) { var syncStartEvent = createSyncStartEvent(); + qdService.sendEvent(syncStartEvent, streamInfo) .then(function () { return fetchGithubPushEvents(userInfo, streamInfo) @@ -113,6 +209,7 @@ module.exports = function (mongoRepository, qdService) { .then(function (events) { return filterEventsToBeSent(events, streamInfo.lastSyncDate); }) + .then(getGithubCommitEvents) .then(convertEventsTo1SelfFormat) .then(function (eventsToBeSent) { return sendEventsToQD(eventsToBeSent, streamInfo); @@ -120,6 +217,9 @@ module.exports = function (mongoRepository, qdService) { .then(function () { var syncCompleteEvent = createSyncCompleteEvent(); return qdService.sendEvent(syncCompleteEvent, streamInfo); + }).catch(function (error) { + console.error("Error occurred", error) }); }; -}; +} +; From 9c7854161dac59c008d54bb5a0884dcde4d070a5 Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Mon, 4 May 2015 17:19:44 +0530 Subject: [PATCH 09/66] Log error --- routes/githubEvents.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 067cac9..81bbe26 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -175,6 +175,7 @@ module.exports = function (mongoRepository, qdService) { deferred.resolve(commit); } else { + console.log("Error occurred :: getCommitPromise", err); deferred.reject(err); } }); From c78ccce5efd556bdec93be968186d703e57ac5aa Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Mon, 4 May 2015 17:30:40 +0100 Subject: [PATCH 10/66] added more tags for sync events so that they can be filtered out in the user aggregations --- routes/githubEvents.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index ed5bd90..6665a0f 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -86,7 +86,7 @@ module.exports = function (mongoRepository, qdService) { var createSyncStartEvent = function () { return { "dateTime": moment().toISOString(), - "objectTags": ["sync"], + "objectTags": ["1self integration sync"], "actionTags": ["start"], "properties": { "source": "GitHub" @@ -96,7 +96,7 @@ module.exports = function (mongoRepository, qdService) { var createSyncCompleteEvent = function () { return { "dateTime": moment().toISOString(), - "objectTags": ["sync"], + "objectTags": ["1self integration sync"], "actionTags": ["complete"], "properties": { "source": "GitHub" From 3f9d08920b13c5c9bdc5ceffacbeac28f41343a8 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 5 May 2015 12:15:46 +0100 Subject: [PATCH 11/66] moving source to be a first class member of the event rather than a property - i'm going to filter on this in the user aggregations --- routes/githubEvents.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 0093228..f6d7699 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -132,8 +132,8 @@ module.exports = function (mongoRepository, qdService) { "dateTime": moment().toISOString(), "objectTags": ["1self integration sync"], "actionTags": ["start"], + "source": "1self-GitHub", "properties": { - "source": "GitHub" } }; }; @@ -142,8 +142,8 @@ module.exports = function (mongoRepository, qdService) { "dateTime": moment().toISOString(), "objectTags": ["1self integration sync"], "actionTags": ["complete"], + "source": "1self-GitHub", "properties": { - "source": "GitHub" } }; }; From 8ad4a658c492e04fced95b7ce042742ce5d57cf8 Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Tue, 5 May 2015 15:21:45 +0530 Subject: [PATCH 12/66] Pass access token for github api requests. --- routes/githubEvents.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index f6d7699..863b11e 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -24,7 +24,7 @@ module.exports = function (mongoRepository, qdService) { }); return deferred.promise; }; - var fetchGithubPushEvents = function (userInfo, streamInfo) { + var fetchGithubPushEvents = function (userInfo) { var deferred = Q.defer(); var pages = _.range(1, 11); var promiseArray = _.map(pages, function (page) { @@ -148,7 +148,7 @@ module.exports = function (mongoRepository, qdService) { }; }; - var getGithubCommitEvents = function (filteredEvents) { + var getGithubCommitEvents = function (filteredEvents, userInfo) { var deferred = Q.defer(); var commitObjects = []; @@ -163,11 +163,12 @@ module.exports = function (mongoRepository, qdService) { console.log("Hitting request") var options = { - url: commitObject.url, + url: commitObject.url + "?access_token=" + userInfo.accessToken, headers: { "User-Agent": "1self" } }; + request(options, function (err, res, body) { if (!err) { var commit = JSON.parse(body); @@ -205,13 +206,14 @@ module.exports = function (mongoRepository, qdService) { qdService.sendEvent(syncStartEvent, streamInfo) .then(function () { - return fetchGithubPushEvents(userInfo, streamInfo) + return fetchGithubPushEvents(userInfo) }) .then(function (events) { return filterEventsToBeSent(events, streamInfo.lastSyncDate); }) - .then(getGithubCommitEvents) - .then(convertEventsTo1SelfFormat) + .then(function(){ + getGithubCommitEvents(userInfo) + }).then(convertEventsTo1SelfFormat) .then(function (eventsToBeSent) { return sendEventsToQD(eventsToBeSent, streamInfo); }) From e7f7db9f35ec98efb07e019422a88814ded3510d Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Tue, 5 May 2015 16:51:54 +0530 Subject: [PATCH 13/66] Add file changes information in GH event. --- routes/githubEvents.js | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 863b11e..d53eec0 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -25,6 +25,7 @@ module.exports = function (mongoRepository, qdService) { return deferred.promise; }; var fetchGithubPushEvents = function (userInfo) { + console.log("fetchGithubPushEvents") var deferred = Q.defer(); var pages = _.range(1, 11); var promiseArray = _.map(pages, function (page) { @@ -45,6 +46,7 @@ module.exports = function (mongoRepository, qdService) { }; var convertEventsTo1SelfFormat = function (filteredEvents) { + var convertEventTo1SelfFormat = function (event) { var clone = function (obj) { return JSON.parse(JSON.stringify(obj)); @@ -99,10 +101,16 @@ module.exports = function (mongoRepository, qdService) { "url": event.commit.url, "line-changes": event.stats.total, "line-additions": event.stats.additions, - "line-deletions": event.stats.deletions - //"file-changes": event.files.changes, - //"file-additions": event.files.additions, - //"file-deletions": event.files.deletions + "line-deletions": event.stats.deletions, + "file-changes": _.reduce(event.files, function (s, e) { + return s + e.changes + }, 0), + "file-additions": _.reduce(event.files, function (s, e) { + return s + e.additions + }, 0), + "file-deletions": _.reduce(event.files, function (s, e) { + return s + e.deletions + }, 0) } }; } else { @@ -110,7 +118,11 @@ module.exports = function (mongoRepository, qdService) { } return clone(singleEventTemplate); }; - return _.map(filteredEvents, convertEventTo1SelfFormat); + var mappedEvents = _.map(filteredEvents, convertEventTo1SelfFormat); + + console.log("MAPPED EVENTS", mappedEvents); + + return mappedEvents; }; var sendEventsToQD = function (events, streamInfo) { @@ -149,6 +161,7 @@ module.exports = function (mongoRepository, qdService) { }; var getGithubCommitEvents = function (filteredEvents, userInfo) { + console.log("In getGithubCommitEvents") var deferred = Q.defer(); var commitObjects = []; @@ -161,9 +174,10 @@ module.exports = function (mongoRepository, qdService) { var getCommitPromise = function (commitObject) { var deferred = Q.defer(); console.log("Hitting request") - + var url = commitObject.url + "?access_token=" + userInfo.accessToken; + console.log("Hitting url", url); var options = { - url: commitObject.url + "?access_token=" + userInfo.accessToken, + url: url, headers: { "User-Agent": "1self" } @@ -185,8 +199,6 @@ module.exports = function (mongoRepository, qdService) { var promiseArray = []; - //commitObjects = commitObjects.slice(0, 3); - _.map(commitObjects, function (commitObject) { promiseArray.push(getCommitPromise(commitObject)) }); @@ -195,7 +207,7 @@ module.exports = function (mongoRepository, qdService) { var events = filteredEvents.concat(commitEvents) deferred.resolve(events); }).catch(function (error) { - console.log("Error occurred", error); + console.log("Error occurred :: getGithubCommitEvents", error); }); return deferred.promise; @@ -211,8 +223,8 @@ module.exports = function (mongoRepository, qdService) { .then(function (events) { return filterEventsToBeSent(events, streamInfo.lastSyncDate); }) - .then(function(){ - getGithubCommitEvents(userInfo) + .then(function (filteredEvents) { + return getGithubCommitEvents(filteredEvents, userInfo) }).then(convertEventsTo1SelfFormat) .then(function (eventsToBeSent) { return sendEventsToQD(eventsToBeSent, streamInfo); @@ -221,7 +233,7 @@ module.exports = function (mongoRepository, qdService) { var syncCompleteEvent = createSyncCompleteEvent(); return qdService.sendEvent(syncCompleteEvent, streamInfo); }).catch(function (error) { - console.error("Error occurred", error) + console.error("Error occurred :: sendGithubEvents ", error) }); }; } From cab66810036f5759a2772acd0cdf5bebb6816362 Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Tue, 5 May 2015 18:14:08 +0530 Subject: [PATCH 14/66] Use proper dates. --- routes/githubEvents.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index d53eec0..3a569aa 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -87,9 +87,9 @@ module.exports = function (mongoRepository, qdService) { "software", "source control" ], - "dateTime": moment(event.created_at).toISOString(), + "dateTime": moment(event.commit.author.date).toISOString(), "latestSyncField": { - "$date": moment(event.created_at).toISOString() + "$date": moment(event.commit.author.date).toISOString() }, "properties": { "pushId": event.pushId, From 2a99a2ae78f710913ebb0212a26f9bcbd0fe4372 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 5 May 2015 17:01:15 +0100 Subject: [PATCH 15/66] Changed the integration uri to come from the context uri; removed oneself_uri as it is a duplicate of context_uri --- routes/githubOAuth.js | 11 ++++++----- routes/qdService.js | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 9b97e20..b6cf60a 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -1,4 +1,4 @@ -var request = require("request"); +Cvar request = require("request"); var passport = require('passport'); var githubStrategy = require('passport-github').Strategy; var _ = require('underscore'); @@ -7,6 +7,8 @@ var GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID; var GITHUB_CLIENT_SECRET = process.env.GITHUB_CLIENT_SECRET; var CONTEXT_URI = process.env.CONTEXT_URI; +var REDIRECT_URL = process.env.CONTEXT_URI + '/integrations'; + module.exports = function (app, mongoRepository, oneselfService) { var handleGithubCallback = function (req, res) { @@ -65,8 +67,7 @@ module.exports = function (app, mongoRepository, oneselfService) { return mongoRepository.update(findQuery, updateQuery) }) .then(function () { - var redirectUrl = process.env.INTEGRATIONS_URI; - res.redirect(redirectUrl); + res.redirect(REDIRECT_URL); }); } else { @@ -78,8 +79,8 @@ module.exports = function (app, mongoRepository, oneselfService) { .replace('{{streamid}}', stream.streamid) .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); syncGithubEvents(callbackUrlForUser, stream.writeToken); - var redirectUrl = process.env.INTEGRATIONS_URI; - res.redirect(redirectUrl); + + res.redirect(REDIRECT_URL); }) }, function (error) { res.render('error', { diff --git a/routes/qdService.js b/routes/qdService.js index 5b00a44..d7d46b0 100644 --- a/routes/qdService.js +++ b/routes/qdService.js @@ -4,7 +4,7 @@ var Q = require('q'); module.exports = function () { var appId = process.env.APP_ID; var appSecret = process.env.APP_SECRET; - var oneselfUri = process.env.ONESELF_URI; + var oneselfUri = process.env.CONTEXT_URI; this.registerStream = function (oneselfUsername, token, callbackUrl) { var deferred = Q.defer(); From 3e2fe3d3afbe634855a529f29e7f8f925b6ba04f Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 5 May 2015 17:26:58 +0100 Subject: [PATCH 16/66] changed context uris to make them clearer --- routes/githubOAuth.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index b6cf60a..fb92a71 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -5,9 +5,8 @@ var _ = require('underscore'); var GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID; var GITHUB_CLIENT_SECRET = process.env.GITHUB_CLIENT_SECRET; -var CONTEXT_URI = process.env.CONTEXT_URI; - -var REDIRECT_URL = process.env.CONTEXT_URI + '/integrations'; +var GITHUB_INT_CONTEXT_URI = process.env.GITHUB_INT_CONTEXT_URI; +var INTEGRATIONS_URL = process.env.CONTEXT_URI + '/integrations'; module.exports = function (app, mongoRepository, oneselfService) { @@ -19,7 +18,7 @@ module.exports = function (app, mongoRepository, oneselfService) { var oneselfUsername = req.session.oneselfUsername; var registrationToken = req.session.registrationToken; console.log("github User is : " + JSON.stringify(githubUser)); - var callbackUrl = CONTEXT_URI + '/authSuccess?username=' + githubUsername + var callbackUrl = GITHUB_INT_CONTEXT_URI + '/authSuccess?username=' + githubUsername + '&latestSyncField={{latestSyncField}}' + '&streamid={{streamid}}'; @@ -67,7 +66,7 @@ module.exports = function (app, mongoRepository, oneselfService) { return mongoRepository.update(findQuery, updateQuery) }) .then(function () { - res.redirect(REDIRECT_URL); + res.redirect(INTEGRATIONS_URL); }); } else { @@ -80,7 +79,7 @@ module.exports = function (app, mongoRepository, oneselfService) { .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); syncGithubEvents(callbackUrlForUser, stream.writeToken); - res.redirect(REDIRECT_URL); + res.redirect(INTEGRATIONS_URL); }) }, function (error) { res.render('error', { @@ -105,7 +104,7 @@ module.exports = function (app, mongoRepository, oneselfService) { passport.use(new githubStrategy({ clientID: GITHUB_CLIENT_ID, clientSecret: GITHUB_CLIENT_SECRET, - callbackURL: CONTEXT_URI + "/auth/github/callback" + callbackURL: GITHUB_INT_CONTEXT_URI + "/auth/github/callback" }, function (accessToken, refreshToken, profile, done) { var githubProfile = { @@ -123,7 +122,7 @@ module.exports = function (app, mongoRepository, oneselfService) { })); app.get('/auth/github/callback', passport.authenticate('github', { - failureRedirect: CONTEXT_URI + failureRedirect: GITHUB_INT_CONTEXT_URI }), handleGithubCallback); } ; From 2e6ef642b333a01443b99d4ebd311acb8bff6922 Mon Sep 17 00:00:00 2001 From: 1Self Team <1self@equalexperts.com> Date: Tue, 5 May 2015 19:52:52 +0000 Subject: [PATCH 17/66] removed typo --- routes/githubOAuth.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index fb92a71..1f707f3 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -1,4 +1,4 @@ -Cvar request = require("request"); +var request = require("request"); var passport = require('passport'); var githubStrategy = require('passport-github').Strategy; var _ = require('underscore'); From c3c91ba085d7a6dbede94be8dac1f4c37898ce9c Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Wed, 6 May 2015 16:01:18 +0530 Subject: [PATCH 18/66] =?UTF-8?q?Consider=20commit=20event=20as=20patch=20?= =?UTF-8?q?if=20committer=20email=20and=20author=20email=20doesn=E2=80=99t?= =?UTF-8?q?=20match.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- routes/githubEvents.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 3a569aa..55934d9 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -113,6 +113,9 @@ module.exports = function (mongoRepository, qdService) { }, 0) } }; + if (event.commit.author.email !== event.commit.committer.email){ + singleEventTemplate.actionTags = ["patch"] + }; } else { console.log("ERROR commit ---->", JSON.stringify(event)) } @@ -173,9 +176,7 @@ module.exports = function (mongoRepository, qdService) { var getCommitPromise = function (commitObject) { var deferred = Q.defer(); - console.log("Hitting request") var url = commitObject.url + "?access_token=" + userInfo.accessToken; - console.log("Hitting url", url); var options = { url: url, headers: { From 48bc6e0032fd5d1468cf7a93dbe9dff1abc207ec Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Wed, 6 May 2015 22:42:07 +0100 Subject: [PATCH 19/66] Changed the github client id and secret names to use the github datasource app --- routes/githubOAuth.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 1f707f3..313521b 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -3,8 +3,8 @@ var passport = require('passport'); var githubStrategy = require('passport-github').Strategy; var _ = require('underscore'); -var GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID; -var GITHUB_CLIENT_SECRET = process.env.GITHUB_CLIENT_SECRET; +var GITHUB_DATASOURCE_CLIENT_ID = process.env.GITHUB_DATASOURCE_CLIENT_ID; +var GITHUB_DATASOURCE_CLIENT_SECRET = process.env.GITHUB_DATASOURCE_CLIENT_SECRET; var GITHUB_INT_CONTEXT_URI = process.env.GITHUB_INT_CONTEXT_URI; var INTEGRATIONS_URL = process.env.CONTEXT_URI + '/integrations'; @@ -102,8 +102,8 @@ module.exports = function (app, mongoRepository, oneselfService) { }); passport.use(new githubStrategy({ - clientID: GITHUB_CLIENT_ID, - clientSecret: GITHUB_CLIENT_SECRET, + clientID: GITHUB_DATASOURCE_CLIENT_ID, + clientSecret: GITHUB_DATASOURCE_CLIENT_SECRET, callbackURL: GITHUB_INT_CONTEXT_URI + "/auth/github/callback" }, function (accessToken, refreshToken, profile, done) { From 334c79d526ded2a96e06baeb39cf078e9003ccc9 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Thu, 7 May 2015 10:53:22 +0100 Subject: [PATCH 20/66] added winston as a logger --- app.js | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/app.js b/app.js index 3c0d1e4..936b1c5 100644 --- a/app.js +++ b/app.js @@ -20,6 +20,30 @@ app.use(bodyParser.urlencoded({ extended: true })); +var winston = require('winston'); +winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); +winston.level = 'info'; +winston.error('Errors will be logged here'); +winston.warn('Warns will be logged here'); +winston.info('Info will be logged here'); +winston.verbose('Verbose will be logged here'); +winston.debug('Debug will be logged here'); +winston.silly('Silly will be logged here'); + + +var logInfo = function(req, username, message, object){ + req.logger.info(username + ': ' + message, object); +} + +var logDebug = function(req, username, message, object){ + req.logger.debug(username + ': ' + message, object); +} + +var logError = function(req, username, message, object){ + req.logger.error(username + ': ' + message, object); +} + + var sessionSecret = process.env.SESSION_SECRET; app.use(session({ secret: sessionSecret, @@ -36,6 +60,12 @@ app.engine('html', swig.renderFile); app.set('views', __dirname + '/views'); app.set('view engine', 'html'); +var attachLogger = function(req, res, next){ + req.logger = winston; + next(); +}; +app.use(attachLogger); + var port = process.env.PORT || 5001; var server = app.listen(port, function () { console.log("Listening on " + port); @@ -60,8 +90,10 @@ mongoClient.connect(mongoUri, function (err, databaseConnection) { }); app.get("/", function (req, res) { + process.env.integrationUrl = req.headers.hostname; req.session.oneselfUsername = req.query.username; req.session.registrationToken = req.query.token; + logInfo(req, req.query.username, 'github setup started: integrationUrl, registrationToken', req.headers.hostname, req.query.token); res.render('index'); }); From a3428179cb32e1ac0b1583bde20a471fc395ecbe Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Thu, 7 May 2015 16:38:35 +0530 Subject: [PATCH 21/66] Remove dependency of some env variables. Refactor code according to that. --- app.js | 9 +++++---- package.json | 5 +++-- routes/githubEvents.js | 12 ++++++------ routes/githubOAuth.js | 18 +++++++++++------- routes/qdService.js | 25 +++++++++++++------------ 5 files changed, 38 insertions(+), 31 deletions(-) diff --git a/app.js b/app.js index 936b1c5..8563790 100644 --- a/app.js +++ b/app.js @@ -90,10 +90,11 @@ mongoClient.connect(mongoUri, function (err, databaseConnection) { }); app.get("/", function (req, res) { - process.env.integrationUrl = req.headers.hostname; + //req.session.integrationUri = req.headers['x-forwarded-host']; + req.session.appUri = req.headers.referer.split("/").slice(0,3).join("/"); req.session.oneselfUsername = req.query.username; req.session.registrationToken = req.query.token; - logInfo(req, req.query.username, 'github setup started: integrationUrl, registrationToken', req.headers.hostname, req.query.token); + logInfo(req, req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); res.render('index'); }); @@ -110,8 +111,8 @@ app.get("/authSuccess", function (req, res) { githubUsername: githubUsername, accessToken: user.accessToken }; - return githubEvents.sendGithubEvents(userInfo, streamInfo); + return githubEvents.sendGithubEvents(userInfo, streamInfo, req.session.appUri); }); res.status(200).send("ok"); } -); \ No newline at end of file +); diff --git a/package.json b/package.json index 1712893..df040ac 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "cookie-parser": "~1.3.2", "express": "4.8.5", "express-session": "1.7.6", + "moment": "~2.8.1", "mongodb": "1.4.8", "morgan": "~1.2.3", "octonode": "~0.6.4", @@ -14,8 +15,8 @@ "q": "1.0.1", "request": "2.40.0", "swig": "~1.4.2", - "moment": "~2.8.1", - "underscore": "^1.7.0" + "underscore": "^1.7.0", + "winston": "^1.0.0" }, "engines": { "node": "0.10.x", diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 55934d9..35b35f9 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -128,12 +128,12 @@ module.exports = function (mongoRepository, qdService) { return mappedEvents; }; - var sendEventsToQD = function (events, streamInfo) { + var sendEventsToQD = function (events, streamInfo, appUri) { var deferred = Q.defer(); if (_.isEmpty(events)) { deferred.resolve(); } - qdService.sendBatchEvents(events, streamInfo) + qdService.sendBatchEvents(events, streamInfo, appUri) .then(function () { console.log("Events sent to 1self!!!"); deferred.resolve(); @@ -214,10 +214,10 @@ module.exports = function (mongoRepository, qdService) { return deferred.promise; }; - this.sendGithubEvents = function (userInfo, streamInfo) { + this.sendGithubEvents = function (userInfo, streamInfo, appUri) { var syncStartEvent = createSyncStartEvent(); - qdService.sendEvent(syncStartEvent, streamInfo) + qdService.sendEvent(syncStartEvent, streamInfo, appUri) .then(function () { return fetchGithubPushEvents(userInfo) }) @@ -228,11 +228,11 @@ module.exports = function (mongoRepository, qdService) { return getGithubCommitEvents(filteredEvents, userInfo) }).then(convertEventsTo1SelfFormat) .then(function (eventsToBeSent) { - return sendEventsToQD(eventsToBeSent, streamInfo); + return sendEventsToQD(eventsToBeSent, streamInfo, appUri); }) .then(function () { var syncCompleteEvent = createSyncCompleteEvent(); - return qdService.sendEvent(syncCompleteEvent, streamInfo); + return qdService.sendEvent(syncCompleteEvent, streamInfo, appUri); }).catch(function (error) { console.error("Error occurred :: sendGithubEvents ", error) }); diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 313521b..2263fe6 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -6,7 +6,6 @@ var _ = require('underscore'); var GITHUB_DATASOURCE_CLIENT_ID = process.env.GITHUB_DATASOURCE_CLIENT_ID; var GITHUB_DATASOURCE_CLIENT_SECRET = process.env.GITHUB_DATASOURCE_CLIENT_SECRET; var GITHUB_INT_CONTEXT_URI = process.env.GITHUB_INT_CONTEXT_URI; -var INTEGRATIONS_URL = process.env.CONTEXT_URI + '/integrations'; module.exports = function (app, mongoRepository, oneselfService) { @@ -28,6 +27,7 @@ module.exports = function (app, mongoRepository, oneselfService) { }; var syncGithubEvents = function (callbackUrl, writeToken) { + request({ method: 'GET', uri: callbackUrl, @@ -41,13 +41,14 @@ module.exports = function (app, mongoRepository, oneselfService) { }; mongoRepository.findByGithubUsername(githubUsername) .then(function (user) { + if (user && user.streamid) { var callbackUrlForUser = callbackUrl .replace('{{streamid}}', user.streamid) .replace('{{latestSyncField}}', user.lastGithubSyncDate.toISOString()); console.log("Syncing github events"); syncGithubEvents(callbackUrlForUser, user.writeToken); - oneselfService.link(oneselfUsername, user.streamid) + oneselfService.link(oneselfUsername, user.streamid, req.session.appUri) .then(function () { var findQuery = { 'githubUsername': githubUsername @@ -66,11 +67,14 @@ module.exports = function (app, mongoRepository, oneselfService) { return mongoRepository.update(findQuery, updateQuery) }) .then(function () { - res.redirect(INTEGRATIONS_URL); - }); + res.redirect(req.session.appUri + "/integrations"); + }) + .catch(function(error){ + console.log("Error occurred", error); + }) } else { - oneselfService.registerStream(oneselfUsername, registrationToken, callbackUrl) + oneselfService.registerStream(oneselfUsername, registrationToken, req.session.appUri, callbackUrl) .then(function (stream) { mongoRepository.insert(document) .then(function () { @@ -78,8 +82,8 @@ module.exports = function (app, mongoRepository, oneselfService) { .replace('{{streamid}}', stream.streamid) .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); syncGithubEvents(callbackUrlForUser, stream.writeToken); - - res.redirect(INTEGRATIONS_URL); + + res.redirect(req.session.appUri + "/integrations"); }) }, function (error) { res.render('error', { diff --git a/routes/qdService.js b/routes/qdService.js index d7d46b0..1859eff 100644 --- a/routes/qdService.js +++ b/routes/qdService.js @@ -2,17 +2,17 @@ var requestModule = require('request'); var Q = require('q'); module.exports = function () { - var appId = process.env.APP_ID; - var appSecret = process.env.APP_SECRET; - var oneselfUri = process.env.CONTEXT_URI; + var appId = process.env.GITHUB_DATASOURCE_APP_ID; + var appSecret = process.env.GITHUB_DATASOURCE_APP_SECRET; + //var oneselfUri = process.env.CONTEXT_URI; - this.registerStream = function (oneselfUsername, token, callbackUrl) { + this.registerStream = function (oneselfUsername, token, appUri, callbackUrl) { var deferred = Q.defer(); - console.log("Registering stream..."); + console.log("Registering stream...", oneselfUsername, token, callbackUrl); var options = { method: 'POST', - uri: oneselfUri + '/v1/users/' + oneselfUsername + '/streams', + uri: appUri + '/v1/users/' + oneselfUsername + '/streams', headers: { 'Authorization': appId + ':' + appSecret, 'registration-token': token @@ -39,11 +39,11 @@ module.exports = function () { return deferred.promise; }; - this.sendBatchEvents = function (events, streamInfo) { + this.sendBatchEvents = function (events, streamInfo, appUri) { var deferred = Q.defer(); var options = { method: 'POST', - uri: oneselfUri + '/v1/streams/' + streamInfo.streamid + '/events/batch', + uri: appUri + '/v1/streams/' + streamInfo.streamid + '/events/batch', gzip: true, headers: { 'Authorization': streamInfo.writeToken, @@ -64,11 +64,11 @@ module.exports = function () { return deferred.promise; }; - this.sendEvent = function (event, streamInfo) { + this.sendEvent = function (event, streamInfo, appUri) { var deferred = Q.defer(); var options = { method: 'POST', - uri: oneselfUri + '/v1/streams/' + streamInfo.streamid + '/events', + uri: appUri + '/v1/streams/' + streamInfo.streamid + '/events', gzip: true, headers: { 'Authorization': streamInfo.writeToken, @@ -88,11 +88,12 @@ module.exports = function () { }); return deferred.promise; }; - this.link = function(oneselfUsername, streamId) { + + this.link = function(oneselfUsername, streamId, appUri) { var deferred = Q.defer(); var options = { method: 'POST', - uri: oneselfUri + '/v1/users/' + oneselfUsername + '/link', + uri: appUri + '/v1/users/' + oneselfUsername + '/link', gzip: true, headers: { 'Content-type': 'application/json' From 9a1c5d03429177660db065e4bef4d1eaaa078217 Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Thu, 7 May 2015 18:37:44 +0530 Subject: [PATCH 22/66] Use proper appUri from process.env --- app.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.js b/app.js index 8563790..451085a 100644 --- a/app.js +++ b/app.js @@ -92,6 +92,7 @@ mongoClient.connect(mongoUri, function (err, databaseConnection) { app.get("/", function (req, res) { //req.session.integrationUri = req.headers['x-forwarded-host']; req.session.appUri = req.headers.referer.split("/").slice(0,3).join("/"); + process.env.appUri = req.session.appUri; req.session.oneselfUsername = req.query.username; req.session.registrationToken = req.query.token; logInfo(req, req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); @@ -111,7 +112,7 @@ app.get("/authSuccess", function (req, res) { githubUsername: githubUsername, accessToken: user.accessToken }; - return githubEvents.sendGithubEvents(userInfo, streamInfo, req.session.appUri); + return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.appUri); }); res.status(200).send("ok"); } From e3da3e49096ceed31e9dc2b25cfe21392a6feb86 Mon Sep 17 00:00:00 2001 From: Douglas Vaz Date: Mon, 11 May 2015 16:22:06 +0530 Subject: [PATCH 23/66] Change tags to single words --- routes/githubEvents.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 35b35f9..b221101 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -61,7 +61,8 @@ module.exports = function (mongoRepository, qdService) { "objectTags": [ "Computer", "Software", - "Source Control" + "Source", + "Control" ], "dateTime": moment(event.created_at).toISOString(), "latestSyncField": { @@ -145,7 +146,7 @@ module.exports = function (mongoRepository, qdService) { var createSyncStartEvent = function () { return { "dateTime": moment().toISOString(), - "objectTags": ["1self integration sync"], + "objectTags": ["1self", "integration", "sync"], "actionTags": ["start"], "source": "1self-GitHub", "properties": { @@ -155,7 +156,7 @@ module.exports = function (mongoRepository, qdService) { var createSyncCompleteEvent = function () { return { "dateTime": moment().toISOString(), - "objectTags": ["1self integration sync"], + "objectTags": ["1self", "integration", "sync"], "actionTags": ["complete"], "source": "1self-GitHub", "properties": { From 46bdb5bd0e5794628bc6144422f98cafe3d765f1 Mon Sep 17 00:00:00 2001 From: Anil Wadghule Date: Thu, 14 May 2015 12:35:05 +0530 Subject: [PATCH 24/66] Fix github commits issue for Douglas and Devaroop. --- routes/githubEvents.js | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index b221101..a5ece86 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -5,6 +5,7 @@ var request = require('request'); var Q = require('q'); module.exports = function (mongoRepository, qdService) { + var getGithubPushEventsPerPage = function (page, userInfo) { var githubUsername = userInfo.githubUsername; var user_api_url = "/users/" + githubUsername; @@ -24,6 +25,7 @@ module.exports = function (mongoRepository, qdService) { }); return deferred.promise; }; + var fetchGithubPushEvents = function (userInfo) { console.log("fetchGithubPushEvents") var deferred = Q.defer(); @@ -47,10 +49,8 @@ module.exports = function (mongoRepository, qdService) { var convertEventsTo1SelfFormat = function (filteredEvents) { - var convertEventTo1SelfFormat = function (event) { - var clone = function (obj) { - return JSON.parse(JSON.stringify(obj)); - }; + var convertEventTo1SelfFormat = function (acc, event) { + if (event.type == "PushEvent") { var singleEventTemplate = { "actionTags": [ @@ -75,6 +75,8 @@ module.exports = function (mongoRepository, qdService) { }) } }; + return acc.concat(singleEventTemplate) + } else if (event.commit != undefined) { var singleEventTemplate = { "actionTags": [ @@ -114,15 +116,19 @@ module.exports = function (mongoRepository, qdService) { }, 0) } }; - if (event.commit.author.email !== event.commit.committer.email){ + if (event.commit.author.email !== event.commit.committer.email) { singleEventTemplate.actionTags = ["patch"] - }; + } + ; + + return acc.concat(singleEventTemplate) } else { console.log("ERROR commit ---->", JSON.stringify(event)) + return acc; + } - return clone(singleEventTemplate); }; - var mappedEvents = _.map(filteredEvents, convertEventTo1SelfFormat); + var mappedEvents = _.reduce(filteredEvents, convertEventTo1SelfFormat, []); console.log("MAPPED EVENTS", mappedEvents); @@ -149,8 +155,7 @@ module.exports = function (mongoRepository, qdService) { "objectTags": ["1self", "integration", "sync"], "actionTags": ["start"], "source": "1self-GitHub", - "properties": { - } + "properties": {} }; }; var createSyncCompleteEvent = function () { @@ -159,8 +164,7 @@ module.exports = function (mongoRepository, qdService) { "objectTags": ["1self", "integration", "sync"], "actionTags": ["complete"], "source": "1self-GitHub", - "properties": { - } + "properties": {} }; }; @@ -169,6 +173,7 @@ module.exports = function (mongoRepository, qdService) { var deferred = Q.defer(); var commitObjects = []; + _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { commitObjects.push({url: commit['url'], pushId: event.payload["push_id"]}) From 032f7278785aaf313cb6cb244da25a2719470ffb Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Wed, 10 Jun 2015 11:35:01 +0100 Subject: [PATCH 25/66] added some logging for github --- app.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/app.js b/app.js index 936b1c5..c74cf49 100644 --- a/app.js +++ b/app.js @@ -30,6 +30,14 @@ winston.verbose('Verbose will be logged here'); winston.debug('Debug will be logged here'); winston.silly('Silly will be logged here'); +winston.info('DBURI=' + process.env.DBURI); +var mongoUri = process.env.DBURI; + +winston.info('PORT=' + process.env.PORT); +var port = process.env.PORT || 5001; + +winston.info('SESSION_SECRET=' + process.env.SESSION_SECRET.substring(0,2) + '...'); +var sessionSecret = process.env.SESSION_SECRET; var logInfo = function(req, username, message, object){ req.logger.info(username + ': ' + message, object); @@ -43,8 +51,6 @@ var logError = function(req, username, message, object){ req.logger.error(username + ': ' + message, object); } - -var sessionSecret = process.env.SESSION_SECRET; app.use(session({ secret: sessionSecret, cookie: { @@ -66,13 +72,12 @@ var attachLogger = function(req, res, next){ }; app.use(attachLogger); -var port = process.env.PORT || 5001; var server = app.listen(port, function () { console.log("Listening on " + port); }); var qdService = new QdService(); -var mongoUri = process.env.DBURI; + var mongoRepository; var githubEvents; var githubOAuth; From 4b95f9154af861e5900f9d9483b10ed7d19692a7 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Wed, 10 Jun 2015 11:36:25 +0100 Subject: [PATCH 26/66] Added timeout and logging --- routes/githubEvents.js | 31 +++++++++++++++++-------------- routes/qdService.js | 4 ++-- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 55934d9..b9e9f67 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -174,7 +174,7 @@ module.exports = function (mongoRepository, qdService) { }) }); - var getCommitPromise = function (commitObject) { + var getCommitPromise = function (commitObject, delay) { var deferred = Q.defer(); var url = commitObject.url + "?access_token=" + userInfo.accessToken; var options = { @@ -184,24 +184,27 @@ module.exports = function (mongoRepository, qdService) { } }; - request(options, function (err, res, body) { - if (!err) { - var commit = JSON.parse(body); - commit.pushId = commitObject.pushId - deferred.resolve(commit); - } - else { - console.log("Error occurred :: getCommitPromise", err); - deferred.reject(err); - } - }); + setTimeout(function(){ + request(options, function (err, res, body) { + if (!err) { + var commit = JSON.parse(body); + commit.pushId = commitObject.pushId + deferred.resolve(commit); + } + else { + console.log("Error occurred :: getCommitPromise", err); + deferred.reject(err); + } + }).end(); + }, + delay); return deferred.promise; }; var promiseArray = []; - _.map(commitObjects, function (commitObject) { - promiseArray.push(getCommitPromise(commitObject)) + _.map(commitObjects, function (commitObject, i) { + promiseArray.push(getCommitPromise(commitObject, i * 100)); }); Q.all(promiseArray).then(function (commitEvents) { diff --git a/routes/qdService.js b/routes/qdService.js index d7d46b0..a8557ce 100644 --- a/routes/qdService.js +++ b/routes/qdService.js @@ -2,8 +2,8 @@ var requestModule = require('request'); var Q = require('q'); module.exports = function () { - var appId = process.env.APP_ID; - var appSecret = process.env.APP_SECRET; + var appId = process.env.GITHUB_APP_ID; + var appSecret = process.env.GITHUB_APP_SECRET; var oneselfUri = process.env.CONTEXT_URI; this.registerStream = function (oneselfUsername, token, callbackUrl) { From db7540af5942febf3c3f7e2de46480cc057b1af1 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Wed, 10 Jun 2015 18:14:53 +0100 Subject: [PATCH 27/66] changed the debug logging line to allow file logging to have a different level of logging to console logging --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index f2dc233..14ce08c 100644 --- a/app.js +++ b/app.js @@ -21,8 +21,8 @@ app.use(bodyParser.urlencoded({ })); var winston = require('winston'); -winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); winston.level = 'info'; +winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); winston.error('Errors will be logged here'); winston.warn('Warns will be logged here'); winston.info('Info will be logged here'); From 5f506626678b75b8cdc9605c88445b50413ae308 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Wed, 10 Jun 2015 18:16:08 +0100 Subject: [PATCH 28/66] changed file related data being added for a commit. The file-changes and so on describes what happened within a file, rather than over all of them. simplified to just say how many files were changed overall --- routes/githubEvents.js | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 745f575..8e7cf2c 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -105,15 +105,7 @@ module.exports = function (mongoRepository, qdService) { "line-changes": event.stats.total, "line-additions": event.stats.additions, "line-deletions": event.stats.deletions, - "file-changes": _.reduce(event.files, function (s, e) { - return s + e.changes - }, 0), - "file-additions": _.reduce(event.files, function (s, e) { - return s + e.additions - }, 0), - "file-deletions": _.reduce(event.files, function (s, e) { - return s + e.deletions - }, 0) + "file-changes": event.files.length) } }; if (event.commit.author.email !== event.commit.committer.email) { @@ -235,7 +227,8 @@ module.exports = function (mongoRepository, qdService) { }) .then(function (filteredEvents) { return getGithubCommitEvents(filteredEvents, userInfo) - }).then(convertEventsTo1SelfFormat) + }) + .then(convertEventsTo1SelfFormat) .then(function (eventsToBeSent) { return sendEventsToQD(eventsToBeSent, streamInfo, appUri); }) From 3623a97829d1e766eba2bba38c2e33856cd9eebf Mon Sep 17 00:00:00 2001 From: 1Self Team <1self@equalexperts.com> Date: Wed, 10 Jun 2015 17:18:08 +0000 Subject: [PATCH 29/66] added extra logging --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index f2dc233..86ffb73 100644 --- a/app.js +++ b/app.js @@ -22,7 +22,7 @@ app.use(bodyParser.urlencoded({ var winston = require('winston'); winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); -winston.level = 'info'; +winston.level = 'debug'; winston.error('Errors will be logged here'); winston.warn('Warns will be logged here'); winston.info('Info will be logged here'); From ddd4dbc1173b76cdbb950c26b4f4ce48a1a3deee Mon Sep 17 00:00:00 2001 From: 1Self Team <1self@equalexperts.com> Date: Wed, 10 Jun 2015 17:21:14 +0000 Subject: [PATCH 30/66] fixed typo --- app.js | 1 - routes/githubEvents.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/app.js b/app.js index 71fa482..41fd3fe 100644 --- a/app.js +++ b/app.js @@ -21,7 +21,6 @@ app.use(bodyParser.urlencoded({ })); var winston = require('winston'); -<<<<<<< HEAD winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); winston.level = 'debug'; winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 8e7cf2c..3488cdb 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -105,7 +105,7 @@ module.exports = function (mongoRepository, qdService) { "line-changes": event.stats.total, "line-additions": event.stats.additions, "line-deletions": event.stats.deletions, - "file-changes": event.files.length) + "file-changes": event.files.length } }; if (event.commit.author.email !== event.commit.committer.email) { From fc1d913f31ab83800530724fdf4c252eec36d3a0 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 12 Jun 2015 17:24:30 +0100 Subject: [PATCH 31/66] Changed the way that commits are counted in the pushes, ids have been moved up into their own top level field, filtering out other people's commits --- app.js | 3 ++- routes/githubEvents.js | 21 +++++++++++++-------- routes/githubOAuth.js | 6 +++++- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/app.js b/app.js index 14ce08c..9546905 100644 --- a/app.js +++ b/app.js @@ -115,7 +115,8 @@ app.get("/authSuccess", function (req, res) { .then(function (user) { var userInfo = { githubUsername: githubUsername, - accessToken: user.accessToken + accessToken: user.accessToken, + displayName: user.displayName }; return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.appUri); }); diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 8e7cf2c..fa22de5 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -64,15 +64,16 @@ module.exports = function (mongoRepository, qdService) { "Source", "Control" ], + "id": event.payload["push_id"], + "childIds": _.map(event.payload.commits, function (c) { + return c['sha'] + }), "dateTime": moment(event.created_at).toISOString(), "latestSyncField": { "$date": moment(event.created_at).toISOString() }, "properties": { - "pushId": event.payload["push_id"], - "commitIds": _.map(event.payload.commits, function (c) { - return c['sha'] - }) + "commits": event.payload.size } }; return acc.concat(singleEventTemplate) @@ -94,18 +95,18 @@ module.exports = function (mongoRepository, qdService) { "latestSyncField": { "$date": moment(event.commit.author.date).toISOString() }, + "id": event.sha, + "parentId": event.pushId, + "url": event.commit.url, "properties": { - "pushId": event.pushId, - "sha": event.sha, "author-name": event.commit.author.name, "author-email": event.commit.author.email, "author-date": event.commit.author.date, "message": event.commit.message, - "url": event.commit.url, "line-changes": event.stats.total, "line-additions": event.stats.additions, "line-deletions": event.stats.deletions, - "file-changes": event.files.length) + "file-changes": event.files.length } }; if (event.commit.author.email !== event.commit.committer.email) { @@ -168,6 +169,10 @@ module.exports = function (mongoRepository, qdService) { _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { + if(commit.author.name !== userInfo.displayName && commit.author.name !== userInfo.githubUsername){ + console.log("" + userInfo.githubUsername + ": ignoring commit for " + commit.author.name); + return; + } commitObjects.push({url: commit['url'], pushId: event.payload["push_id"]}) }) }); diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 2263fe6..795f68f 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -21,9 +21,12 @@ module.exports = function (app, mongoRepository, oneselfService) { + '&latestSyncField={{latestSyncField}}' + '&streamid={{streamid}}'; + // we use the display name to spot commits that weren't made by the committer. + // this can happen when a rebase occurs. var document = { githubUsername: githubUsername, - accessToken: req.user.accessToken + accessToken: req.user.accessToken, + displayName: req.user.profile.displayName, }; var syncGithubEvents = function (callbackUrl, writeToken) { @@ -105,6 +108,7 @@ module.exports = function (app, mongoRepository, oneselfService) { done(null, obj); }); + console passport.use(new githubStrategy({ clientID: GITHUB_DATASOURCE_CLIENT_ID, clientSecret: GITHUB_DATASOURCE_CLIENT_SECRET, From 7f4f86b28e10b78faf134888ad3ee85a7df69d25 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Mon, 15 Jun 2015 17:11:48 +0100 Subject: [PATCH 32/66] Added a break down of number of files changes per file type --- routes/githubEvents.js | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index fa22de5..ccf1684 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -3,6 +3,7 @@ var github = require('octonode'); var moment = require('moment'); var request = require('request'); var Q = require('q'); +var path = require('path'); module.exports = function (mongoRepository, qdService) { @@ -73,12 +74,22 @@ module.exports = function (mongoRepository, qdService) { "$date": moment(event.created_at).toISOString() }, "properties": { - "commits": event.payload.size + "commits": event.payload.size, + "repo": event.repo.name } }; return acc.concat(singleEventTemplate) - } else if (event.commit != undefined) { + } + else if (event.commit != undefined) { + var extensionStats = _.reduce(event.files, function(result, file){ + var ext = path.extname(file.filename).substring(1); + result[ext] = result[ext] || {}; + result[ext]['line-additions'] = (result[ext]['line-additions'] || 0) + file['additions']; + result[ext]['line-deletions'] = (result[ext]['line-deletions'] || 0) + file['deletions']; + result[ext]['line-changes'] = (result[ext]['line-changes'] || 0) + file['changes']; + return result; + }, {}); var singleEventTemplate = { "actionTags": [ "commit" @@ -106,16 +117,19 @@ module.exports = function (mongoRepository, qdService) { "line-changes": event.stats.total, "line-additions": event.stats.additions, "line-deletions": event.stats.deletions, - "file-changes": event.files.length + "file-changes": event.files.length, + "repo": event.repo, + "file-types": extensionStats } - }; + } + if (event.commit.author.email !== event.commit.committer.email) { singleEventTemplate.actionTags = ["patch"] } - ; - + return acc.concat(singleEventTemplate) - } else { + } + else { console.log("ERROR commit ---->", JSON.stringify(event)) return acc; @@ -173,7 +187,7 @@ module.exports = function (mongoRepository, qdService) { console.log("" + userInfo.githubUsername + ": ignoring commit for " + commit.author.name); return; } - commitObjects.push({url: commit['url'], pushId: event.payload["push_id"]}) + commitObjects.push({url: commit['url'], pushId: event.payload["push_id"], repo: event.repo.name}) }) }); @@ -191,7 +205,8 @@ module.exports = function (mongoRepository, qdService) { request(options, function (err, res, body) { if (!err) { var commit = JSON.parse(body); - commit.pushId = commitObject.pushId + commit.pushId = commitObject.pushId; + commit.repo = commitObject.repo; deferred.resolve(commit); } else { From c248b5273f45727efa03f150fc66866a5f54d063 Mon Sep 17 00:00:00 2001 From: 1Self Team <1self@equalexperts.com> Date: Tue, 16 Jun 2015 11:52:58 +0000 Subject: [PATCH 33/66] removed erroneous logging line --- app.js | 1 - 1 file changed, 1 deletion(-) diff --git a/app.js b/app.js index 04d8fe4..e530128 100644 --- a/app.js +++ b/app.js @@ -23,7 +23,6 @@ app.use(bodyParser.urlencoded({ var winston = require('winston'); winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); winston.level = 'debug'; -winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); winston.error('Errors will be logged here'); winston.warn('Warns will be logged here'); winston.info('Info will be logged here'); From c94627d49478ff7ecfc6ca7bc3e4452a707dfd1d Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 16 Jun 2015 14:05:19 +0100 Subject: [PATCH 34/66] Added logging of all evetns --- app.js | 12 ++++---- routes/githubEvents.js | 67 ++++++++++++++++++++++++++++++------------ 2 files changed, 55 insertions(+), 24 deletions(-) diff --git a/app.js b/app.js index e530128..311885f 100644 --- a/app.js +++ b/app.js @@ -21,8 +21,7 @@ app.use(bodyParser.urlencoded({ })); var winston = require('winston'); -winston.add(winston.transports.File, { filename: 'github-datasource.log', level: 'debug', json: false }); -winston.level = 'debug'; +winston.add(winston.transports.File, { filename: 'github.log', level: 'debug', json: false }); winston.error('Errors will be logged here'); winston.warn('Warns will be logged here'); winston.info('Info will be logged here'); @@ -39,6 +38,8 @@ var port = process.env.PORT || 5001; winston.info('SESSION_SECRET=' + process.env.SESSION_SECRET.substring(0,2) + '...'); var sessionSecret = process.env.SESSION_SECRET; +var logger = winston; + var logInfo = function(req, username, message, object){ req.logger.info(username + ': ' + message, object); } @@ -73,7 +74,7 @@ var attachLogger = function(req, res, next){ app.use(attachLogger); var server = app.listen(port, function () { - console.log("Listening on " + port); + logger.info("Listening on " + port); }); var qdService = new QdService(); @@ -83,11 +84,10 @@ var githubEvents; var githubOAuth; mongoClient.connect(mongoUri, function (err, databaseConnection) { if (err) { - console.error("Could not connect to Mongodb with URI : " + mongoUri); - console.error(err); + logger.error("Could not connect to Mongodb with URI: ", [mongoUri, err]); process.exit(1); } else { - console.log("connected to mongo : ", mongoUri); + logger.info("connected to mongo: ", mongoUri); mongoRepository = new MongoRepository(databaseConnection); githubEvents = new GithubEvents(mongoRepository, qdService); githubOAuth = new GithubOAuth(app, mongoRepository, qdService); diff --git a/routes/githubEvents.js b/routes/githubEvents.js index ccf1684..5aa06f3 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -4,10 +4,24 @@ var moment = require('moment'); var request = require('request'); var Q = require('q'); var path = require('path'); +var logger = require('winston'); + +var logInfo = function(username, message, object){ + logger.info(username + ': ' + message, object); +} + +var logDebug = function(username, message, object){ + logger.debug(username + ': ' + message, object); +} + +var logError = function(username, message, object){ + logger.error(username + ': ' + message, object); +} module.exports = function (mongoRepository, qdService) { var getGithubPushEventsPerPage = function (page, userInfo) { + logDebug(userInfo.githubUsername, "fetching page: ", page); var githubUsername = userInfo.githubUsername; var user_api_url = "/users/" + githubUsername; var client = github.client(userInfo.accessToken); @@ -18,7 +32,7 @@ module.exports = function (mongoRepository, qdService) { var deferred = Q.defer(); githubUser.events(page, ['PushEvent'], function (err, pushEvents) { if (err) { - console.log("err " + err); + logError(githubUsername, "err " + err); deferred.reject(err); } else { deferred.resolve(pushEvents); @@ -28,7 +42,7 @@ module.exports = function (mongoRepository, qdService) { }; var fetchGithubPushEvents = function (userInfo) { - console.log("fetchGithubPushEvents") + logInfo(userInfo.githubUsername, "fetchGithubPushEvents", userInfo) var deferred = Q.defer(); var pages = _.range(1, 11); var promiseArray = _.map(pages, function (page) { @@ -48,10 +62,10 @@ module.exports = function (mongoRepository, qdService) { return _.filter(events, eventsToBeSent); }; - var convertEventsTo1SelfFormat = function (filteredEvents) { - + var convertEventsTo1SelfFormat = function (filteredEvents, username) { + logDebug(username, 'converting events to 1self format'); var convertEventTo1SelfFormat = function (acc, event) { - + logDebug(username, 'raw event: ', event); if (event.type == "PushEvent") { var singleEventTemplate = { "actionTags": [ @@ -78,10 +92,13 @@ module.exports = function (mongoRepository, qdService) { "repo": event.repo.name } }; - return acc.concat(singleEventTemplate) + + logDebug(username, 'converted push event: ', singleEventTemplate); + return acc.concat(singleEventTemplate); } else if (event.commit != undefined) { + logDebug(username, 'raw event', event); var extensionStats = _.reduce(event.files, function(result, file){ var ext = path.extname(file.filename).substring(1); result[ext] = result[ext] || {}; @@ -126,11 +143,14 @@ module.exports = function (mongoRepository, qdService) { if (event.commit.author.email !== event.commit.committer.email) { singleEventTemplate.actionTags = ["patch"] } + + logDebug(username, 'converted commit event: ', singleEventTemplate); return acc.concat(singleEventTemplate) } else { - console.log("ERROR commit ---->", JSON.stringify(event)) + logError(username, 'Error commit', event); + logDebug(username, 'Error event', event); return acc; } @@ -142,16 +162,17 @@ module.exports = function (mongoRepository, qdService) { return mappedEvents; }; - var sendEventsToQD = function (events, streamInfo, appUri) { + var sendEventsToQD = function (events, streamInfo, appUri, userInfo) { var deferred = Q.defer(); if (_.isEmpty(events)) { deferred.resolve(); } qdService.sendBatchEvents(events, streamInfo, appUri) .then(function () { - console.log("Events sent to 1self!!!"); + logInfo(userInfo.githubUsername, 'Events sent to 1self', [events.length, streamInfo]); deferred.resolve(); }, function (error) { + logError(userInfo.githubUsername, 'Error while sending events', error); deferred.reject(error); }); return deferred.promise; @@ -176,7 +197,7 @@ module.exports = function (mongoRepository, qdService) { }; var getGithubCommitEvents = function (filteredEvents, userInfo) { - console.log("In getGithubCommitEvents") + logDebug(userInfo.githubUsername, 'getting commit events, push events length: ', filteredEvents.length); var deferred = Q.defer(); var commitObjects = []; @@ -184,13 +205,15 @@ module.exports = function (mongoRepository, qdService) { _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { if(commit.author.name !== userInfo.displayName && commit.author.name !== userInfo.githubUsername){ - console.log("" + userInfo.githubUsername + ": ignoring commit for " + commit.author.name); + logDebug(userInfo.githubUsername, 'ignoring commit for ' + commit.author.name); return; } commitObjects.push({url: commit['url'], pushId: event.payload["push_id"], repo: event.repo.name}) }) }); + logDebug(userInfo.githubUsername, 'commit requests created: ', commitObjects); + var getCommitPromise = function (commitObject, delay) { var deferred = Q.defer(); var url = commitObject.url + "?access_token=" + userInfo.accessToken; @@ -202,6 +225,7 @@ module.exports = function (mongoRepository, qdService) { }; setTimeout(function(){ + logDebug(userInfo.githubUsername, 'executing commit object request: ', options); request(options, function (err, res, body) { if (!err) { var commit = JSON.parse(body); @@ -210,7 +234,7 @@ module.exports = function (mongoRepository, qdService) { deferred.resolve(commit); } else { - console.log("Error occurred :: getCommitPromise", err); + logDebug(githubUsername.username, 'Error occurred getting commit: options, err: ', [options, err]); deferred.reject(err); } }).end(); @@ -226,10 +250,11 @@ module.exports = function (mongoRepository, qdService) { }); Q.all(promiseArray).then(function (commitEvents) { + logDebug(userInfo.githubUsername, 'all commit events retrieved'); var events = filteredEvents.concat(commitEvents) deferred.resolve(events); }).catch(function (error) { - console.log("Error occurred :: getGithubCommitEvents", error); + logError(userInfo.githubUsername, 'Error occurred :: getGithubCommitEvents', error); }); return deferred.promise; @@ -246,17 +271,23 @@ module.exports = function (mongoRepository, qdService) { return filterEventsToBeSent(events, streamInfo.lastSyncDate); }) .then(function (filteredEvents) { - return getGithubCommitEvents(filteredEvents, userInfo) + return getGithubCommitEvents(filteredEvents, userInfo); + }) + .then(function (filteredEvents){ + convertEventsTo1SelfFormat(filteredEvents, userInfo.githubUsername); }) - .then(convertEventsTo1SelfFormat) .then(function (eventsToBeSent) { - return sendEventsToQD(eventsToBeSent, streamInfo, appUri); + return sendEventsToQD(eventsToBeSent, streamInfo, appUri, userInfo); }) .then(function () { var syncCompleteEvent = createSyncCompleteEvent(); return qdService.sendEvent(syncCompleteEvent, streamInfo, appUri); - }).catch(function (error) { - console.error("Error occurred :: sendGithubEvents ", error) + }) + .then(function() { + logInfo(userInfo, 'finished sending events to 1self'); + }) + .catch(function (error) { + logDebug(userInfo.githubUsername, 'Error occurred :: sendGithubEvents', error); }); }; } From 7431dd7e3f278e7139b75bfd03e1d18a327c0acf Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 16 Jun 2015 14:19:18 +0100 Subject: [PATCH 35/66] added extra logging to ignoring events --- routes/githubEvents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 5aa06f3..01693bd 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -205,7 +205,7 @@ module.exports = function (mongoRepository, qdService) { _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { if(commit.author.name !== userInfo.displayName && commit.author.name !== userInfo.githubUsername){ - logDebug(userInfo.githubUsername, 'ignoring commit for ' + commit.author.name); + logDebug(userInfo.githubUsername, 'ignoring commit for ' + commit.author.name, commit.author); return; } commitObjects.push({url: commit['url'], pushId: event.payload["push_id"], repo: event.repo.name}) From 1bd4cc03348e380e744e715e837b3a6f1fe9b789 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 16 Jun 2015 14:31:00 +0100 Subject: [PATCH 36/66] Added commit and push to the logging --- routes/githubEvents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 01693bd..1498657 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -205,7 +205,7 @@ module.exports = function (mongoRepository, qdService) { _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { if(commit.author.name !== userInfo.displayName && commit.author.name !== userInfo.githubUsername){ - logDebug(userInfo.githubUsername, 'ignoring commit for ' + commit.author.name, commit.author); + logDebug(userInfo.githubUsername, 'ignoring commit for ' + commit.author.name, [commit.author, commit, event]); return; } commitObjects.push({url: commit['url'], pushId: event.payload["push_id"], repo: event.repo.name}) From 47d614ac32892ce9ad1fbf09b30e406e856666da Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 16 Jun 2015 14:43:32 +0100 Subject: [PATCH 37/66] Added userinfo to the logging --- routes/githubEvents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 1498657..64b6ee5 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -205,7 +205,7 @@ module.exports = function (mongoRepository, qdService) { _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { if(commit.author.name !== userInfo.displayName && commit.author.name !== userInfo.githubUsername){ - logDebug(userInfo.githubUsername, 'ignoring commit for ' + commit.author.name, [commit.author, commit, event]); + logDebug(userInfo.githubUsername, 'ignoring commit for ' + commit.author.name, [userInfo, commit.author, commit, event]); return; } commitObjects.push({url: commit['url'], pushId: event.payload["push_id"], repo: event.repo.name}) From f213e7a290a75e0089a059b657421513942dd4ff Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 16 Jun 2015 15:09:22 +0100 Subject: [PATCH 38/66] changed merge detection to be based on commit author --- routes/githubEvents.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 64b6ee5..b8b1f4d 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -204,8 +204,8 @@ module.exports = function (mongoRepository, qdService) { _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { - if(commit.author.name !== userInfo.displayName && commit.author.name !== userInfo.githubUsername){ - logDebug(userInfo.githubUsername, 'ignoring commit for ' + commit.author.name, [userInfo, commit.author, commit, event]); + if(commit.author.name !== commit.committer.name){ + logDebug(userInfo.githubUsername, 'ignoring commit'[userInfo, commit.author, commit.committer, commit, event]); return; } commitObjects.push({url: commit['url'], pushId: event.payload["push_id"], repo: event.repo.name}) From 2500f157c34ac4b7cb0310253c53e62f5eaba4c0 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 16 Jun 2015 23:52:48 +0100 Subject: [PATCH 39/66] Implemented email address lookup and checking against commits to differentiate between commits and merges --- app.js | 3 +- routes/githubEvents.js | 27 +++++-- routes/githubOAuth.js | 169 +++++++++++++++++++++++++---------------- routes/qdService.js | 13 ++-- 4 files changed, 134 insertions(+), 78 deletions(-) diff --git a/app.js b/app.js index 311885f..fd83e39 100644 --- a/app.js +++ b/app.js @@ -116,7 +116,8 @@ app.get("/authSuccess", function (req, res) { var userInfo = { githubUsername: githubUsername, accessToken: user.accessToken, - displayName: user.displayName + displayName: user.displayName, + email: user.email }; return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.appUri); }); diff --git a/routes/githubEvents.js b/routes/githubEvents.js index b8b1f4d..75d4f9c 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -63,6 +63,7 @@ module.exports = function (mongoRepository, qdService) { }; var convertEventsTo1SelfFormat = function (filteredEvents, username) { + var deferred = Q.defer(); logDebug(username, 'converting events to 1self format'); var convertEventTo1SelfFormat = function (acc, event) { logDebug(username, 'raw event: ', event); @@ -140,6 +141,10 @@ module.exports = function (mongoRepository, qdService) { } } + if(event.committerIsAuthor === false){ + singleEventTemplate.actionTags = ["merge"]; + } + if (event.commit.author.email !== event.commit.committer.email) { singleEventTemplate.actionTags = ["patch"] } @@ -202,13 +207,22 @@ module.exports = function (mongoRepository, qdService) { var commitObjects = []; + var userEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(userInfo.email); + var userEmail = userEmailParts[1] + userEmailParts[3]; + _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { - if(commit.author.name !== commit.committer.name){ - logDebug(userInfo.githubUsername, 'ignoring commit'[userInfo, commit.author, commit.committer, commit, event]); - return; + var commitReq = { + url: commit['url'], + pushId: event.payload["push_id"], + repo: event.repo.name } - commitObjects.push({url: commit['url'], pushId: event.payload["push_id"], repo: event.repo.name}) + + var commitEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(commit.author.email); + var commitEmail = commitEmailParts[1] + commitEmailParts[3]; + commitReq.committerIsAuthor = commitEmail === userEmail; + logDebug(userInfo.githubUsername, 'commit req', commitReq); + commitObjects.push(commitReq); }) }); @@ -231,10 +245,11 @@ module.exports = function (mongoRepository, qdService) { var commit = JSON.parse(body); commit.pushId = commitObject.pushId; commit.repo = commitObject.repo; + commit.committerIsAuthor = commitObject.committerIsAuthor; deferred.resolve(commit); } else { - logDebug(githubUsername.username, 'Error occurred getting commit: options, err: ', [options, err]); + logDebug(userInfo.githubUsername, 'Error occurred getting commit: options, err: ', [options, err]); deferred.reject(err); } }).end(); @@ -274,7 +289,7 @@ module.exports = function (mongoRepository, qdService) { return getGithubCommitEvents(filteredEvents, userInfo); }) .then(function (filteredEvents){ - convertEventsTo1SelfFormat(filteredEvents, userInfo.githubUsername); + return convertEventsTo1SelfFormat(filteredEvents, userInfo.githubUsername); }) .then(function (eventsToBeSent) { return sendEventsToQD(eventsToBeSent, streamInfo, appUri, userInfo); diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index bd3a0fd..a7112d4 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -2,15 +2,29 @@ var request = require("request"); var passport = require('passport'); var githubStrategy = require('passport-github').Strategy; var _ = require('underscore'); +var q = require('q'); +var logger = require('winston'); var GITHUB_DATASOURCE_CLIENT_ID = process.env.GITHUB_DATASOURCE_CLIENT_ID; var GITHUB_DATASOURCE_CLIENT_SECRET = process.env.GITHUB_DATASOURCE_CLIENT_SECRET; var GITHUB_INT_CONTEXT_URI = process.env.GITHUB_INT_CONTEXT_URI; -console.log('env'); -console.log(GITHUB_DATASOURCE_CLIENT_ID); -console.log(GITHUB_DATASOURCE_CLIENT_SECRET); -console.log(GITHUB_INT_CONTEXT_URI); +var logInfo = function(username, message, object){ + logger.info(username + ': ' + message, object); +} + +var logDebug = function(username, message, object){ + logger.debug(username + ': ' + message, object); +} + +var logError = function(username, message, object){ + logger.error(username + ': ' + message, object); +} + +logger.info('env'); +logger.info(GITHUB_DATASOURCE_CLIENT_ID); +logger.info(GITHUB_DATASOURCE_CLIENT_SECRET); +logger.info(GITHUB_INT_CONTEXT_URI); module.exports = function (app, mongoRepository, oneselfService) { @@ -21,19 +35,41 @@ module.exports = function (app, mongoRepository, oneselfService) { req.session.githubUsername = githubUsername; var oneselfUsername = req.session.oneselfUsername; var registrationToken = req.session.registrationToken; - console.log("github User is : " + JSON.stringify(githubUser)); + logDebug(githubUsername, "github User is: " + JSON.stringify(githubUser)); var callbackUrl = GITHUB_INT_CONTEXT_URI + '/authSuccess?username=' + githubUsername + '&latestSyncField={{latestSyncField}}' + '&streamid={{streamid}}'; // we use the display name to spot commits that weren't made by the committer. // this can happen when a rebase occurs. - var document = { + var userInfo = { githubUsername: githubUsername, accessToken: req.user.accessToken, displayName: req.user.profile.displayName, }; + var getEmailAddress = function (accessToken) { + var deferred = q.defer(); + var options = { + url: "https://api.github.com/user/emails?access_token=" + accessToken, + headers: { + "User-Agent": "1self" + } + }; + request(options, function (err, res, body) { + if (!err) { + userInfo.email = JSON.parse(body)[0].email; + logDebug(userInfo.githubUsername, 'email retrieved from users github profile', userInfo.email); + deferred.resolve(); + } + else { + logDebug(userInfo.githubUsername, 'error trying to get email: ', err); + deferred.reject(err); + } + }); + return deferred.promise; + }; + var syncGithubEvents = function (callbackUrl, writeToken) { request({ @@ -44,65 +80,68 @@ module.exports = function (app, mongoRepository, oneselfService) { 'Authorization': writeToken } }, function (e, response, body) { - console.log("Synced github Events!!!"); + console.log("Started event sync"); }); }; - mongoRepository.findByGithubUsername(githubUsername) - .then(function (user) { - - if (user && user.streamid) { - var callbackUrlForUser = callbackUrl - .replace('{{streamid}}', user.streamid) - .replace('{{latestSyncField}}', user.lastGithubSyncDate.toISOString()); - console.log("Syncing github events"); - syncGithubEvents(callbackUrlForUser, user.writeToken); - oneselfService.link(oneselfUsername, user.streamid, req.session.appUri) - .then(function () { - var findQuery = { - 'githubUsername': githubUsername - }; - var updateQuery = { - "$set": { - "accessToken": req.user.accessToken - }, - "$unset": { - "streamid": 1, - "readToken": 1, - "writeToken": 1, - "lastGithubSyncDate": 1 - } - }; - return mongoRepository.update(findQuery, updateQuery) - }) - .then(function () { - res.redirect(req.session.appUri + "/integrations"); - }) - .catch(function(error){ - console.log("Error occurred", error); - }) - } - else { - oneselfService.registerStream(oneselfUsername, registrationToken, req.session.appUri, callbackUrl) - .then(function (stream) { - mongoRepository.insert(document) - .then(function () { - var callbackUrlForUser = callbackUrl - .replace('{{streamid}}', stream.streamid) - .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); - syncGithubEvents(callbackUrlForUser, stream.writeToken); - - res.redirect(req.session.appUri + "/integrations"); - }) - }, function (error) { - res.render('error', { - error: error - }); - }) - } - }) - .catch(function (error) { - console.error("Error in github callback: ", error); - }); + + getEmailAddress(req.session.accessToken) + .then(function(){ + mongoRepository.findByGithubUsername(userInfo.githubUsername); + }) + .then(function (user) { + if (user && user.streamid) { + var callbackUrlForUser = callbackUrl + .replace('{{streamid}}', user.streamid) + .replace('{{latestSyncField}}', user.lastGithubSyncDate.toISOString()); + console.log("Syncing github events"); + syncGithubEvents(callbackUrlForUser, user.writeToken); + oneselfService.link(oneselfUsername, user.streamid, req.session.appUri) + .then(function () { + var findQuery = { + 'githubUsername': githubUsername + }; + var updateQuery = { + "$set": { + "accessToken": req.user.accessToken + }, + "$unset": { + "streamid": 1, + "readToken": 1, + "writeToken": 1, + "lastGithubSyncDate": 1 + } + }; + return mongoRepository.update(findQuery, updateQuery) + }) + .then(function () { + res.redirect(req.session.appUri + "/integrations"); + }) + .catch(function(error){ + logError(userInfo.githubUsername, "Error occurred", error); + }) + } + else { + oneselfService.registerStream(oneselfUsername, registrationToken, req.session.appUri, callbackUrl) + .then(function (stream) { + mongoRepository.insert(userInfo) + .then(function () { + var callbackUrlForUser = callbackUrl + .replace('{{streamid}}', stream.streamid) + .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); + syncGithubEvents(callbackUrlForUser, stream.writeToken); + + res.redirect(req.session.appUri + "/integrations"); + }) + }, function (error) { + res.render('error', { + error: error + }); + }) + } + }) + .catch(function (error) { + logError(userInfo.githubUsername, "Error in github callback: ", error); + }); }; passport.serializeUser(function (user, done) { @@ -133,11 +172,11 @@ module.exports = function (app, mongoRepository, oneselfService) { app.use(passport.session()); app.get('/auth/github', passport.authenticate('github', { - scope: 'repo' + scope: 'repo,user:email' })); app.get('/auth/github/callback', passport.authenticate('github', { - failureRedirect: GITHUB_INT_CONTEXT_URI + failureRedirect: GITHUB_INT_CONTEXT_URI, }), handleGithubCallback); } ; diff --git a/routes/qdService.js b/routes/qdService.js index 784b8a6..abf3c67 100644 --- a/routes/qdService.js +++ b/routes/qdService.js @@ -1,21 +1,22 @@ var requestModule = require('request'); var Q = require('q'); +var logger = require('winston'); module.exports = function () { var appId = process.env.GITHUB_APP_ID; var appSecret = process.env.GITHUB_APP_SECRET; var oneselfUri = process.env.CONTEXT_URI; - console.log('1self service env'); - console.log(process.env.GITHUB_APP_ID); - console.log(process.env.GITHUB_APP_SECRET); - console.log(process.env.CONTEXT_URI); + logger.info('1self service env'); + logger.info(process.env.GITHUB_APP_ID); + logger.info(process.env.GITHUB_APP_SECRET); + logger.info(process.env.CONTEXT_URI); this.registerStream = function (oneselfUsername, token, appUri, callbackUrl) { var deferred = Q.defer(); - console.log("Registering stream...", oneselfUsername, token, callbackUrl, appUri); + logger.debug("Registering stream...", oneselfUsername, token, callbackUrl, appUri); var requestUri = appUri + '/v1/users/' + oneselfUsername + '/streams'; - console.log(requestUri); + logger.debug(requestUri); var options = { method: 'POST', uri: appUri + '/v1/users/' + oneselfUsername + '/streams', From dd19759a4c89b8788b600b725b7053d81706c0d7 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Thu, 6 Aug 2015 17:28:07 +0100 Subject: [PATCH 40/66] implemented the ability to reauth for a github user --- .gitignore | 3 + app.js | 24 ++++++- routes/githubOAuth.js | 138 ++++++++++++++++++++++++-------------- views/reauth.html | 20 ++++++ views/reauthcomplete.html | 18 +++++ 5 files changed, 152 insertions(+), 51 deletions(-) create mode 100644 views/reauth.html create mode 100644 views/reauthcomplete.html diff --git a/.gitignore b/.gitignore index 3f82a04..31b28c8 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ node_modules .idea .env + +#mac file system +.DS_Store diff --git a/app.js b/app.js index fd83e39..558c831 100644 --- a/app.js +++ b/app.js @@ -96,14 +96,34 @@ mongoClient.connect(mongoUri, function (err, databaseConnection) { app.get("/", function (req, res) { //req.session.integrationUri = req.headers['x-forwarded-host']; - req.session.appUri = req.headers.referer.split("/").slice(0,3).join("/"); - process.env.appUri = req.session.appUri; + req.session.appUri = req.headers.referer === undefined ? '/' : req.headers.referer.split("/").slice(0,3).join("/"); + process.env.appUri = req.session.appUri; req.session.oneselfUsername = req.query.username; req.session.registrationToken = req.query.token; logInfo(req, req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); res.render('index'); }); +app.get("/reauth", function (req, res) { + //req.session.integrationUri = req.headers['x-forwarded-host']; + req.session.appUri = process.env.CONTEXT_URI; + req.session.redirect = '/reauth/complete'; + req.session.reauth = true; + req.session.username = req.query.username; + logInfo(req, req.query.username, 'reauth page hit, ', [req.session.appUri, req.query.token]); + res.render('reauth'); +}); + +app.get("/reauth/complete", function (req, res) { + //req.session.integrationUri = req.headers['x-forwarded-host']; + req.session.appUri = null; + req.session.redirect = null + req.session.reauth = null; + req.session.username = null; + logInfo(req, req.query.username, 'reauth complete page served, ', [req.session.appUri, req.query.token]); + res.render('reauthcomplete'); +}); + app.get("/authSuccess", function (req, res) { var githubUsername = req.query.username; var streamInfo = { diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index a7112d4..6538767 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -28,9 +28,10 @@ logger.info(GITHUB_INT_CONTEXT_URI); module.exports = function (app, mongoRepository, oneselfService) { - var handleGithubCallback = function (req, res) { + var handleNewAuthCallback = function(req, res){ var githubUser = req.user.profile; var githubUsername = githubUser.username; + req.session.accessToken = req.user.accessToken; req.session.githubUsername = githubUsername; var oneselfUsername = req.session.oneselfUsername; @@ -86,62 +87,101 @@ module.exports = function (app, mongoRepository, oneselfService) { getEmailAddress(req.session.accessToken) .then(function(){ - mongoRepository.findByGithubUsername(userInfo.githubUsername); + return mongoRepository.findByGithubUsername(userInfo.githubUsername); }) .then(function (user) { - if (user && user.streamid) { - var callbackUrlForUser = callbackUrl - .replace('{{streamid}}', user.streamid) - .replace('{{latestSyncField}}', user.lastGithubSyncDate.toISOString()); - console.log("Syncing github events"); - syncGithubEvents(callbackUrlForUser, user.writeToken); - oneselfService.link(oneselfUsername, user.streamid, req.session.appUri) - .then(function () { - var findQuery = { - 'githubUsername': githubUsername - }; - var updateQuery = { - "$set": { - "accessToken": req.user.accessToken - }, - "$unset": { - "streamid": 1, - "readToken": 1, - "writeToken": 1, - "lastGithubSyncDate": 1 - } - }; - return mongoRepository.update(findQuery, updateQuery) - }) - .then(function () { - res.redirect(req.session.appUri + "/integrations"); - }) - .catch(function(error){ - logError(userInfo.githubUsername, "Error occurred", error); - }) + // we hit this if we are doing a re-auth and the user already exists + oneselfService.registerStream(oneselfUsername, registrationToken, req.session.appUri, callbackUrl) + .then(function (stream) { + mongoRepository.insert(userInfo) + .then(function () { + var callbackUrlForUser = callbackUrl + .replace('{{streamid}}', stream.streamid) + .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); + syncGithubEvents(callbackUrlForUser, stream.writeToken); + + res.redirect(req.session.appUri + "/integrations"); + }) + }, function (error) { + res.render('error', { + error: error + }); + }) + }) + .catch(function (error) { + logError(userInfo.githubUsername, "Error in github callback: ", error); + }); + } + + var handleReauthCallback = function(req, res){ + var githubUser = req.user.profile; + var githubUsername = githubUser.username; + + logDebug(githubUsername, "github User is: " + JSON.stringify(githubUser)); + var callbackUrl = GITHUB_INT_CONTEXT_URI + '/authSuccess?username=' + githubUsername + + '&latestSyncField={{latestSyncField}}' + + '&streamid={{streamid}}'; + + var userInfo = { + githubUsername: githubUsername, + accessToken: req.user.accessToken, + displayName: req.user.profile.displayName, + }; + + var getEmailAddress = function (accessToken) { + var deferred = q.defer(); + var options = { + url: "https://api.github.com/user/emails?access_token=" + accessToken, + headers: { + "User-Agent": "1self" + } + }; + request(options, function (err, res, body) { + if (!err) { + userInfo.email = JSON.parse(body)[0].email; + logDebug(userInfo.githubUsername, 'email retrieved from users github profile', userInfo.email); + deferred.resolve(); + } + else { + logDebug(userInfo.githubUsername, 'error trying to get email: ', err); + deferred.reject(err); + } + }); + return deferred.promise; + }; + + getEmailAddress(userInfo.accessToken) + .then(function(){ + return mongoRepository.findByGithubUsername(githubUsername); + }) + .then(function (user) { + var query = { + _id: user._id } - else { - oneselfService.registerStream(oneselfUsername, registrationToken, req.session.appUri, callbackUrl) - .then(function (stream) { - mongoRepository.insert(userInfo) - .then(function () { - var callbackUrlForUser = callbackUrl - .replace('{{streamid}}', stream.streamid) - .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); - syncGithubEvents(callbackUrlForUser, stream.writeToken); - - res.redirect(req.session.appUri + "/integrations"); - }) - }, function (error) { - res.render('error', { - error: error - }); - }) + + var operation = { + $set: { + accessToken: userInfo.accessToken, + email: userInfo.email + } } + + mongoRepository.update(query, operation) + .then(function () { + res.redirect(req.session.redirect); + }) }) .catch(function (error) { logError(userInfo.githubUsername, "Error in github callback: ", error); }); + } + var handleGithubCallback = function (req, res) { + if(req.session.reauth){ + handleReauthCallback(req, res); + } + else{ + handleNewAuthCallback(req, res); + } }; passport.serializeUser(function (user, done) { diff --git a/views/reauth.html b/views/reauth.html new file mode 100644 index 0000000..be0bdb0 --- /dev/null +++ b/views/reauth.html @@ -0,0 +1,20 @@ + + + + + + + + 1self Github Integration + + +
+

Github Integration for 1self

+

Reauthorize the github plugin

+ +
+ + + + + \ No newline at end of file diff --git a/views/reauthcomplete.html b/views/reauthcomplete.html new file mode 100644 index 0000000..212fc2b --- /dev/null +++ b/views/reauthcomplete.html @@ -0,0 +1,18 @@ + + + + + + + + 1self Github Integration + + +
+

Reauth successful

+
+ + + + + \ No newline at end of file From 02f43a9aa9b2ecb8dad1f4255c8d7b01c57517ff Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 13:35:39 +0100 Subject: [PATCH 41/66] added logging to the github plugin --- app.js | 123 +++++++++++++++-------- routes/.eslintrc | 5 + routes/githubEvents.js | 219 ++++++++++++++++++++++------------------- routes/githubOAuth.js | 2 +- 4 files changed, 205 insertions(+), 144 deletions(-) create mode 100644 routes/.eslintrc diff --git a/app.js b/app.js index 558c831..a456da7 100644 --- a/app.js +++ b/app.js @@ -1,5 +1,6 @@ -var express = require("express"); -var session = require("express-session"); +/*jslint node: true */ +var express = require('express'); +var session = require('express-session'); var path = require('path'); var swig = require('swig'); var q = require('q'); @@ -8,12 +9,16 @@ var bodyParser = require('body-parser'); var cookieParser = require('cookie-parser'); var mongoClient = require('mongodb').MongoClient; -var GithubEvents = require("./routes/githubEvents"); +var GithubEvents = require('./routes/githubEvents'); var MongoRepository = require('./routes/mongoRepository'); -var GithubOAuth = require("./routes/githubOAuth"); -var QdService = require("./routes/qdService"); +var GithubOAuth = require('./routes/githubOAuth'); +var QdService = require('./routes/qdService'); var app = express(); + +/*jslint nomen: true */ app.use(express.static(path.join(__dirname, 'public'))); +/*jslint nomen: false */ + app.use(logger()); app.use(cookieParser()); app.use(bodyParser.urlencoded({ @@ -40,16 +45,40 @@ var sessionSecret = process.env.SESSION_SECRET; var logger = winston; -var logInfo = function(req, username, message, object){ - req.logger.info(username + ': ' + message, object); +logger.logInfo = function(username, message, object){ + if(object) { + logger.info(username + ': ' + message, object); + } + else { + logger.info(username + ': ' + message, []); + } } -var logDebug = function(req, username, message, object){ - req.logger.debug(username + ': ' + message, object); +logger.logDebug = function(username, message, object){ + if(object) { + logger.debug(username + ': ' + message, object); + } + else { + logger.debug(username + ': ' + message, []); + } } -var logError = function(req, username, message, object){ - req.logger.error(username + ': ' + message, object); +logger.logError = function(username, message, object){ + if(object) { + logger.error(username + ': ' + message, object); + } + else { + logger.error(username + ': ' + message, []); + } +} + +logger.logSilly = function(username, message, object){ + if(object) { + logger.silly(username + ': ' + message, object); + } + else { + logger.silly(username + ': ' + message, []); + } } app.use(session({ @@ -74,7 +103,7 @@ var attachLogger = function(req, res, next){ app.use(attachLogger); var server = app.listen(port, function () { - logger.info("Listening on " + port); + logger.logInfo('', 'Listening on ' + port); }); var qdService = new QdService(); @@ -84,63 +113,77 @@ var githubEvents; var githubOAuth; mongoClient.connect(mongoUri, function (err, databaseConnection) { if (err) { - logger.error("Could not connect to Mongodb with URI: ", [mongoUri, err]); + logger.logError('Could not connect to Mongodb with URI: ', [mongoUri, err]); process.exit(1); } else { - logger.info("connected to mongo: ", mongoUri); + logger.logInfo('connected to mongo: ', mongoUri); mongoRepository = new MongoRepository(databaseConnection); githubEvents = new GithubEvents(mongoRepository, qdService); githubOAuth = new GithubOAuth(app, mongoRepository, qdService); + + githubEvents.setLogger(logger); } }); -app.get("/", function (req, res) { +app.get('/', function (req, res) { //req.session.integrationUri = req.headers['x-forwarded-host']; - req.session.appUri = req.headers.referer === undefined ? '/' : req.headers.referer.split("/").slice(0,3).join("/"); + req.session.appUri = req.headers.referer === undefined ? '/' : req.headers.referer.split('/').slice(0,3).join('/'); process.env.appUri = req.session.appUri; req.session.oneselfUsername = req.query.username; req.session.registrationToken = req.query.token; - logInfo(req, req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); + logger.logInfo(req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); res.render('index'); }); -app.get("/reauth", function (req, res) { +app.get('/reauth', function (req, res) { //req.session.integrationUri = req.headers['x-forwarded-host']; req.session.appUri = process.env.CONTEXT_URI; req.session.redirect = '/reauth/complete'; req.session.reauth = true; req.session.username = req.query.username; - logInfo(req, req.query.username, 'reauth page hit, ', [req.session.appUri, req.query.token]); + logger.logInfo(req, req.query.username, 'reauthing', [req.session.appUri, req.query.token]); + logger.logInfo(req, req.query.username, 'serving reauth page'); res.render('reauth'); }); -app.get("/reauth/complete", function (req, res) { +app.get('/reauth/complete', function (req, res) { //req.session.integrationUri = req.headers['x-forwarded-host']; req.session.appUri = null; req.session.redirect = null req.session.reauth = null; req.session.username = null; - logInfo(req, req.query.username, 'reauth complete page served, ', [req.session.appUri, req.query.token]); + logger.logInfo(req, req.query.username, 'reauthing complete, ', [req.session.appUri, req.query.token]); + logger.logDebug(req, req.query.username, 'serving reauth complete page'); res.render('reauthcomplete'); }); -app.get("/authSuccess", function (req, res) { - var githubUsername = req.query.username; - var streamInfo = { - streamid: req.query.streamid, - writeToken: req.headers.authorization, - lastSyncDate: req.query.latestSyncField - }; - mongoRepository.findByGithubUsername(githubUsername) - .then(function (user) { - var userInfo = { - githubUsername: githubUsername, - accessToken: user.accessToken, - displayName: user.displayName, - email: user.email - }; - return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.appUri); - }); - res.status(200).send("ok"); +// This is the entry point after auth, to do a sync. It's also used +// as the callback url in the stream +app.get('/authSuccess', function (req, res) { + var githubUsername = req.query.username; + var streamInfo = { + streamid: req.query.streamid, + writeToken: req.headers.authorization, + lastSyncDate: req.query.latestSyncField + }; + logger.logInfo(req.query.username, 'syncing data, [stream id, write token]', [streamInfo.streamid.substring(0, 2), streamInfo.writeToken.substring(0, 2)]); + + mongoRepository.findByGithubUsername(githubUsername) + .then(function(user) { + logger.logDebug(user.githubUsername, 'found in the database'); + var userInfo = { + githubUsername: githubUsername, + accessToken: user.accessToken, + displayName: user.displayName, + email: user.email + }; + + return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.appUri); + }) + .catch(function(error){ + logger.logError(githubUsername, 'sync requested for unknown user'); + }); + + res.status(200).send('ok, sync request acknowledged'); } -); +); \ No newline at end of file diff --git a/routes/.eslintrc b/routes/.eslintrc new file mode 100644 index 0000000..bb3c020 --- /dev/null +++ b/routes/.eslintrc @@ -0,0 +1,5 @@ +{ + "env": { + "node": true + } +} diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 75d4f9c..ee48c4d 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -1,30 +1,20 @@ -var _ = require("underscore"); +var _ = require('underscore'); var github = require('octonode'); var moment = require('moment'); var request = require('request'); var Q = require('q'); var path = require('path'); -var logger = require('winston'); - -var logInfo = function(username, message, object){ - logger.info(username + ': ' + message, object); -} - -var logDebug = function(username, message, object){ - logger.debug(username + ': ' + message, object); -} - -var logError = function(username, message, object){ - logger.error(username + ': ' + message, object); -} module.exports = function (mongoRepository, qdService) { + var logger; + var getGithubPushEventsPerPage = function (page, userInfo) { - logDebug(userInfo.githubUsername, "fetching page: ", page); + var githubUsername = userInfo.githubUsername; - var user_api_url = "/users/" + githubUsername; + var user_api_url = '/users/' + githubUsername; var client = github.client(userInfo.accessToken); + logger.logDebug(userInfo.githubUsername, 'fetching page [page, user_api_url, accessToken]', [page, user_api_url, userInfo.accessToken.substring(0,2)]); client.get(user_api_url, {}, function (err, status, body, headers) { }); var githubUser = client.user(githubUsername); @@ -32,7 +22,7 @@ module.exports = function (mongoRepository, qdService) { var deferred = Q.defer(); githubUser.events(page, ['PushEvent'], function (err, pushEvents) { if (err) { - logError(githubUsername, "err " + err); + logger.logError(githubUsername, 'err ' + err); deferred.reject(err); } else { deferred.resolve(pushEvents); @@ -42,7 +32,6 @@ module.exports = function (mongoRepository, qdService) { }; var fetchGithubPushEvents = function (userInfo) { - logInfo(userInfo.githubUsername, "fetchGithubPushEvents", userInfo) var deferred = Q.defer(); var pages = _.range(1, 11); var promiseArray = _.map(pages, function (page) { @@ -55,51 +44,53 @@ module.exports = function (mongoRepository, qdService) { }); return deferred.promise; }; - var filterEventsToBeSent = function (events, lastSyncDate) { + + var filterEventsToBeSent = function (userInfo, events, lastSyncDate) { + logger.logDebug(userInfo.githubUsername, 'filtering events, [event count, lastSyncDate]', [events.length, lastSyncDate]); var eventsToBeSent = function (event) { return moment(event.created_at).isAfter(lastSyncDate); }; - return _.filter(events, eventsToBeSent); + var result = _.filter(events, eventsToBeSent); + logger.logDebug(userInfo.githubUsername, 'events filtered, [event count]', [events.length]); + return result; }; var convertEventsTo1SelfFormat = function (filteredEvents, username) { var deferred = Q.defer(); - logDebug(username, 'converting events to 1self format'); var convertEventTo1SelfFormat = function (acc, event) { - logDebug(username, 'raw event: ', event); - if (event.type == "PushEvent") { + logger.logSilly(username, 'converting raw event: ', event); + if (event.type == 'PushEvent') { var singleEventTemplate = { - "actionTags": [ - "Github", - "Push" + 'actionTags': [ + 'Github', + 'Push' ], - "source": "GitHub", - "objectTags": [ - "Computer", - "Software", - "Source", - "Control" + 'source': 'GitHub', + 'objectTags': [ + 'Computer', + 'Software', + 'Source', + 'Control' ], - "id": event.payload["push_id"], - "childIds": _.map(event.payload.commits, function (c) { + 'id': event.payload['push_id'], + 'childIds': _.map(event.payload.commits, function (c) { return c['sha'] }), - "dateTime": moment(event.created_at).toISOString(), - "latestSyncField": { - "$date": moment(event.created_at).toISOString() + 'dateTime': moment(event.created_at).toISOString(), + 'latestSyncField': { + '$date': moment(event.created_at).toISOString() }, - "properties": { - "commits": event.payload.size, - "repo": event.repo.name + 'properties': { + 'commits': event.payload.size, + 'repo': event.repo.name } }; - logDebug(username, 'converted push event: ', singleEventTemplate); + logger.logSilly(username, 'converted push event: ', singleEventTemplate); return acc.concat(singleEventTemplate); } else if (event.commit != undefined) { - logDebug(username, 'raw event', event); var extensionStats = _.reduce(event.files, function(result, file){ var ext = path.extname(file.filename).substring(1); result[ext] = result[ext] || {}; @@ -109,60 +100,58 @@ module.exports = function (mongoRepository, qdService) { return result; }, {}); var singleEventTemplate = { - "actionTags": [ - "commit" + 'actionTags': [ + 'commit' ], - "source": "GitHub", - "objectTags": [ - "git", - "github", - "computer", - "software", - "source control" + 'source': 'GitHub', + 'objectTags': [ + 'git', + 'github', + 'computer', + 'software', + 'source control' ], - "dateTime": moment(event.commit.author.date).toISOString(), - "latestSyncField": { - "$date": moment(event.commit.author.date).toISOString() + 'dateTime': moment(event.commit.author.date).toISOString(), + 'latestSyncField': { + '$date': moment(event.commit.author.date).toISOString() }, - "id": event.sha, - "parentId": event.pushId, - "url": event.commit.url, - "properties": { - "author-name": event.commit.author.name, - "author-email": event.commit.author.email, - "author-date": event.commit.author.date, - "message": event.commit.message, - "line-changes": event.stats.total, - "line-additions": event.stats.additions, - "line-deletions": event.stats.deletions, - "file-changes": event.files.length, - "repo": event.repo, - "file-types": extensionStats + 'id': event.sha, + 'parentId': event.pushId, + 'url': event.commit.url, + 'properties': { + 'author-name': event.commit.author.name, + 'author-email': event.commit.author.email, + 'author-date': event.commit.author.date, + 'message': event.commit.message, + 'line-changes': event.stats.total, + 'line-additions': event.stats.additions, + 'line-deletions': event.stats.deletions, + 'file-changes': event.files.length, + 'repo': event.repo, + 'file-types': extensionStats } } if(event.committerIsAuthor === false){ - singleEventTemplate.actionTags = ["merge"]; + singleEventTemplate.actionTags = ['merge']; } if (event.commit.author.email !== event.commit.committer.email) { - singleEventTemplate.actionTags = ["patch"] + singleEventTemplate.actionTags = ['patch'] } - logDebug(username, 'converted commit event: ', singleEventTemplate); + logger.logSilly(username, 'converted commit event: ', singleEventTemplate); return acc.concat(singleEventTemplate) } else { - logError(username, 'Error commit', event); - logDebug(username, 'Error event', event); + logger.logSilly(username, 'couldnt convert as event is not a push or commit'); return acc; - } }; - var mappedEvents = _.reduce(filteredEvents, convertEventTo1SelfFormat, []); - console.log("MAPPED EVENTS", mappedEvents); + var mappedEvents = _.reduce(filteredEvents, convertEventTo1SelfFormat, []); + logger.logDebug(username, 'mapped events to 1self format, [event count]', mappedEvents.length); return mappedEvents; }; @@ -170,39 +159,44 @@ module.exports = function (mongoRepository, qdService) { var sendEventsToQD = function (events, streamInfo, appUri, userInfo) { var deferred = Q.defer(); if (_.isEmpty(events)) { + logger.logDebug(userInfo.githubUsername, 'there are no events to send'); deferred.resolve(); } + + logger.logDebug(userInfo.githubUsername, 'sending events to qd [event count, app uri]', [events.length, appUri]); qdService.sendBatchEvents(events, streamInfo, appUri) .then(function () { - logInfo(userInfo.githubUsername, 'Events sent to 1self', [events.length, streamInfo]); + logger.logDebug(userInfo.githubUsername, 'events sent to 1self, [event count, stream info]', [events.length, streamInfo]); deferred.resolve(); }, function (error) { - logError(userInfo.githubUsername, 'Error while sending events', error); + logger.logError(userInfo.githubUsername, 'error while sending events', error); deferred.reject(error); }); return deferred.promise; }; + var createSyncStartEvent = function () { return { - "dateTime": moment().toISOString(), - "objectTags": ["1self", "integration", "sync"], - "actionTags": ["start"], - "source": "1self-GitHub", - "properties": {} + 'dateTime': moment().toISOString(), + 'objectTags': ['1self', 'integration', 'sync'], + 'actionTags': ['start'], + 'source': '1self-GitHub', + 'properties': {} }; }; + var createSyncCompleteEvent = function () { return { - "dateTime": moment().toISOString(), - "objectTags": ["1self", "integration", "sync"], - "actionTags": ["complete"], - "source": "1self-GitHub", - "properties": {} + 'dateTime': moment().toISOString(), + 'objectTags': ['1self', 'integration', 'sync'], + 'actionTags': ['complete'], + 'source': '1self-GitHub', + 'properties': {} }; }; var getGithubCommitEvents = function (filteredEvents, userInfo) { - logDebug(userInfo.githubUsername, 'getting commit events, push events length: ', filteredEvents.length); + logger.logDebug(userInfo.githubUsername, 'getting commit events for push events, [event count]', filteredEvents.length); var deferred = Q.defer(); var commitObjects = []; @@ -210,38 +204,45 @@ module.exports = function (mongoRepository, qdService) { var userEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(userInfo.email); var userEmail = userEmailParts[1] + userEmailParts[3]; + logger.logDebug(userInfo.githubUsername, 'flattening commits'); + logger.logDebug(userInfo.githubUsername, 'removing push email alias, [original, unaliased]', [userInfo.email, userEmail]); + + _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { var commitReq = { url: commit['url'], - pushId: event.payload["push_id"], + pushId: event.payload['push_id'], repo: event.repo.name } var commitEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(commit.author.email); var commitEmail = commitEmailParts[1] + commitEmailParts[3]; commitReq.committerIsAuthor = commitEmail === userEmail; - logDebug(userInfo.githubUsername, 'commit req', commitReq); + logger.logDebug(userInfo.githubUsername, 'commit req', commitReq); commitObjects.push(commitReq); }) }); - logDebug(userInfo.githubUsername, 'commit requests created: ', commitObjects); + logger.logDebug(userInfo.githubUsername, 'commit have been flattened, [commits]', commitObjects); + // the delay is in there as the github api is returning an error when hit with lots of requests + // quickly. var getCommitPromise = function (commitObject, delay) { var deferred = Q.defer(); - var url = commitObject.url + "?access_token=" + userInfo.accessToken; + var url = commitObject.url + '?access_token=' + userInfo.accessToken; var options = { url: url, headers: { - "User-Agent": "1self" + 'User-Agent': '1self' } }; setTimeout(function(){ - logDebug(userInfo.githubUsername, 'executing commit object request: ', options); + logger.logDebug(userInfo.githubUsername, [options.url, 'requesting commit, [options, delay]'].join(': '), [options, delay]); request(options, function (err, res, body) { if (!err) { + logger.logDebug(userInfo.githubUsername, [options.url, 'request successful'].join(': ')); var commit = JSON.parse(body); commit.pushId = commitObject.pushId; commit.repo = commitObject.repo; @@ -249,7 +250,7 @@ module.exports = function (mongoRepository, qdService) { deferred.resolve(commit); } else { - logDebug(userInfo.githubUsername, 'Error occurred getting commit: options, err: ', [options, err]); + logger.logDebug(userInfo.githubUsername, [options.url, 'request failed, [error]'].join(': '), err); deferred.reject(err); } }).end(); @@ -265,45 +266,57 @@ module.exports = function (mongoRepository, qdService) { }); Q.all(promiseArray).then(function (commitEvents) { - logDebug(userInfo.githubUsername, 'all commit events retrieved'); + logger.logDebug(userInfo.githubUsername, 'all commit events retrieved'); var events = filteredEvents.concat(commitEvents) deferred.resolve(events); }).catch(function (error) { - logError(userInfo.githubUsername, 'Error occurred :: getGithubCommitEvents', error); + logger.logError(userInfo.githubUsername, 'Error occurred :: getGithubCommitEvents', error); }); return deferred.promise; }; this.sendGithubEvents = function (userInfo, streamInfo, appUri) { + logger.logInfo(userInfo.githubUsername, 'starting sync, sending start sync event'); var syncStartEvent = createSyncStartEvent(); qdService.sendEvent(syncStartEvent, streamInfo, appUri) .then(function () { + logger.logInfo(userInfo.githubUsername, 'fetching events from github api', []); return fetchGithubPushEvents(userInfo) }) .then(function (events) { - return filterEventsToBeSent(events, streamInfo.lastSyncDate); + logger.logInfo(userInfo.githubUsername, 'filtering events to include only pushes, [event count]', [events.length]); + return filterEventsToBeSent(userInfo, events, streamInfo.lastSyncDate); }) .then(function (filteredEvents) { + logger.logInfo(userInfo.githubUsername, 'getting the commit events for the pushes, [event count]', filteredEvents.length); return getGithubCommitEvents(filteredEvents, userInfo); }) .then(function (filteredEvents){ + logger.logInfo(userInfo.githubUsername, 'converting events to 1self format, [event count]', filteredEvents.length); return convertEventsTo1SelfFormat(filteredEvents, userInfo.githubUsername); }) .then(function (eventsToBeSent) { + logger.logInfo(userInfo.githubUsername, 'sending the events to 1self, [event count]', eventsToBeSent.length); return sendEventsToQD(eventsToBeSent, streamInfo, appUri, userInfo); }) .then(function () { + logger.logInfo(userInfo.githubUsername, 'sending sync complete to 1self', []); var syncCompleteEvent = createSyncCompleteEvent(); return qdService.sendEvent(syncCompleteEvent, streamInfo, appUri); }) .then(function() { - logInfo(userInfo, 'finished sending events to 1self'); + logger.logInfo(userInfo.githubUsername, 'finished sending events to 1self'); }) .catch(function (error) { - logDebug(userInfo.githubUsername, 'Error occurred :: sendGithubEvents', error); + logger.logError(userInfo.githubUsername, 'Error occurred :: sendGithubEvents', error); }); }; -} -; + + this.setLogger = function(newLogger){ + logger = newLogger; + } + + +}; diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 6538767..4ec5395 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -117,7 +117,7 @@ module.exports = function (app, mongoRepository, oneselfService) { var githubUser = req.user.profile; var githubUsername = githubUser.username; - logDebug(githubUsername, "github User is: " + JSON.stringify(githubUser)); + logDebug(githubUsername, "github User is: ", JSON.stringify(githubUser)); var callbackUrl = GITHUB_INT_CONTEXT_URI + '/authSuccess?username=' + githubUsername + '&latestSyncField={{latestSyncField}}' + '&streamid={{streamid}}'; From 1a1cdfd742b2299fc0d93ff9ee6b7b1e48adc67a Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 15:21:16 +0100 Subject: [PATCH 42/66] moved the data that never changes into an info field --- routes/githubEvents.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index ee48c4d..cfc4051 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -118,11 +118,13 @@ module.exports = function (mongoRepository, qdService) { 'id': event.sha, 'parentId': event.pushId, 'url': event.commit.url, - 'properties': { + 'info': { + 'message': event.commit.message, 'author-name': event.commit.author.name, 'author-email': event.commit.author.email, - 'author-date': event.commit.author.date, - 'message': event.commit.message, + 'author-date': event.commit.author.date + }, + 'properties': { 'line-changes': event.stats.total, 'line-additions': event.stats.additions, 'line-deletions': event.stats.deletions, From 15d2f5d3a1628befc99975c9c9146750e9394f35 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 15:31:25 +0100 Subject: [PATCH 43/66] made the logging use the env var LOGGINGDIR in the filename --- app.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app.js b/app.js index a456da7..1b24fbf 100644 --- a/app.js +++ b/app.js @@ -25,8 +25,11 @@ app.use(bodyParser.urlencoded({ extended: true })); + var winston = require('winston'); -winston.add(winston.transports.File, { filename: 'github.log', level: 'debug', json: false }); +var filename = process.env.LOGGINGDIR ? path.join(process.env.LOGGINGDIR, 'github.log') : 'github.log'; +winston.info('logging debug messages to ' + filename); +winston.add(winston.transports.File, { filename: filename, level: 'debug', json: false }); winston.error('Errors will be logged here'); winston.warn('Warns will be logged here'); winston.info('Info will be logged here'); From d2314afc4d773e0af1b0ac657d5555356b29622f Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 15:38:59 +0100 Subject: [PATCH 44/66] removed request object from logging statements as it was undefined --- app.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app.js b/app.js index 1b24fbf..cafc237 100644 --- a/app.js +++ b/app.js @@ -99,7 +99,7 @@ app.engine('html', swig.renderFile); app.set('views', __dirname + '/views'); app.set('view engine', 'html'); -var attachLogger = function(req, res, next){ +var attachLogger = function, res, next){ req.logger = winston; next(); }; @@ -144,8 +144,8 @@ app.get('/reauth', function (req, res) { req.session.redirect = '/reauth/complete'; req.session.reauth = true; req.session.username = req.query.username; - logger.logInfo(req, req.query.username, 'reauthing', [req.session.appUri, req.query.token]); - logger.logInfo(req, req.query.username, 'serving reauth page'); + logger.logInfo(req.query.username, 'reauthing', [req.session.appUri, req.query.token]); + logger.logInfo(req.query.username, 'serving reauth page'); res.render('reauth'); }); @@ -155,8 +155,8 @@ app.get('/reauth/complete', function (req, res) { req.session.redirect = null req.session.reauth = null; req.session.username = null; - logger.logInfo(req, req.query.username, 'reauthing complete, ', [req.session.appUri, req.query.token]); - logger.logDebug(req, req.query.username, 'serving reauth complete page'); + logger.logInfo(req.query.username, 'reauthing complete, ', [req.session.appUri, req.query.token]); + logger.logDebug(req.query.username, 'serving reauth complete page'); res.render('reauthcomplete'); }); From 59ae779d5e3c53c51954d7c27aea048d95f4a689 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 15:46:35 +0100 Subject: [PATCH 45/66] corrected typo --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index cafc237..ce28c33 100644 --- a/app.js +++ b/app.js @@ -99,7 +99,7 @@ app.engine('html', swig.renderFile); app.set('views', __dirname + '/views'); app.set('view engine', 'html'); -var attachLogger = function, res, next){ +var attachLogger = function(req, res, next){ req.logger = winston; next(); }; From 8f4178ad06025929d90963d34c671a1bbbde5176 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 15:56:11 +0100 Subject: [PATCH 46/66] detecting errors in stream request --- routes/qdService.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/routes/qdService.js b/routes/qdService.js index abf3c67..1f775d8 100644 --- a/routes/qdService.js +++ b/routes/qdService.js @@ -30,6 +30,11 @@ module.exports = function () { } }; requestModule(options, function (e, response, body) { + if (e) { + deferred.reject("Error: ", e); + return; + } + if (response.statusCode === 401) { deferred.reject('auth error: check your appId and appSecret', null); return; @@ -39,9 +44,7 @@ module.exports = function () { deferred.reject('Invalid username and registrationToken', null); return; } - if (e) { - deferred.reject("Error: ", e); - } + deferred.resolve(body); }); return deferred.promise; From 6e998b6a303c50f63b616beb77ce4251b69ce422 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 16:09:05 +0100 Subject: [PATCH 47/66] added page that handles when the reauth isn't given a user --- app.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app.js b/app.js index ce28c33..4b30a48 100644 --- a/app.js +++ b/app.js @@ -139,6 +139,11 @@ app.get('/', function (req, res) { }); app.get('/reauth', function (req, res) { + if(req.query.username === undefined){ + res.render('reauthUnknownUser'); + return; + } + //req.session.integrationUri = req.headers['x-forwarded-host']; req.session.appUri = process.env.CONTEXT_URI; req.session.redirect = '/reauth/complete'; From f1f4104353c1c7405c889db09ec9b35dbc624f90 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 17:11:51 +0100 Subject: [PATCH 48/66] added logging of error when sync request fails --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index 4b30a48..84b937c 100644 --- a/app.js +++ b/app.js @@ -189,7 +189,7 @@ app.get('/authSuccess', function (req, res) { return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.appUri); }) .catch(function(error){ - logger.logError(githubUsername, 'sync requested for unknown user'); + logger.logError(githubUsername, 'sync requested for unknown user', error); }); res.status(200).send('ok, sync request acknowledged'); From 6bb927a9969380205b9563eec4353f49ff1f8ece Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 17:25:07 +0100 Subject: [PATCH 49/66] adding apprui and reauth for unkown user --- app.js | 4 +++- routes/githubOAuth.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app.js b/app.js index 84b937c..687e750 100644 --- a/app.js +++ b/app.js @@ -43,6 +43,8 @@ var mongoUri = process.env.DBURI; winston.info('PORT=' + process.env.PORT); var port = process.env.PORT || 5001; +winston.info('APIURI=' + process.env.APIURI); + winston.info('SESSION_SECRET=' + process.env.SESSION_SECRET.substring(0,2) + '...'); var sessionSecret = process.env.SESSION_SECRET; @@ -131,7 +133,7 @@ mongoClient.connect(mongoUri, function (err, databaseConnection) { app.get('/', function (req, res) { //req.session.integrationUri = req.headers['x-forwarded-host']; req.session.appUri = req.headers.referer === undefined ? '/' : req.headers.referer.split('/').slice(0,3).join('/'); - process.env.appUri = req.session.appUri; + process.env.appUri = req.session.appUri; req.session.oneselfUsername = req.query.username; req.session.registrationToken = req.query.token; logger.logInfo(req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index 4ec5395..d455b6c 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -91,7 +91,7 @@ module.exports = function (app, mongoRepository, oneselfService) { }) .then(function (user) { // we hit this if we are doing a re-auth and the user already exists - oneselfService.registerStream(oneselfUsername, registrationToken, req.session.appUri, callbackUrl) + oneselfService.registerStream(oneselfUsername, registrationToken, process.env.APIURI, callbackUrl) .then(function (stream) { mongoRepository.insert(userInfo) .then(function () { From 1153545f918e3f208da9dfd0c55b8947091a68d6 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 17:32:49 +0100 Subject: [PATCH 50/66] correcting apiuri --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index 687e750..1846f9a 100644 --- a/app.js +++ b/app.js @@ -188,7 +188,7 @@ app.get('/authSuccess', function (req, res) { email: user.email }; - return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.appUri); + return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.APPURI); }) .catch(function(error){ logger.logError(githubUsername, 'sync requested for unknown user', error); From 8664fe0b1b74418fdeff566f35ca0a573aea780d Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 17:36:24 +0100 Subject: [PATCH 51/66] added logging of uri when sending start event --- routes/githubEvents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index cfc4051..6a56a97 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -279,7 +279,7 @@ module.exports = function (mongoRepository, qdService) { }; this.sendGithubEvents = function (userInfo, streamInfo, appUri) { - logger.logInfo(userInfo.githubUsername, 'starting sync, sending start sync event'); + logger.logInfo(userInfo.githubUsername, 'starting sync, sending start sync event, [appUri]', [appUri]); var syncStartEvent = createSyncStartEvent(); qdService.sendEvent(syncStartEvent, streamInfo, appUri) From 27daf5d8e77f4508989d1ec4464c4a76229a14b7 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 7 Aug 2015 17:41:29 +0100 Subject: [PATCH 52/66] corrected appkey to apikey --- app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.js b/app.js index 1846f9a..457ed4b 100644 --- a/app.js +++ b/app.js @@ -188,10 +188,10 @@ app.get('/authSuccess', function (req, res) { email: user.email }; - return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.APPURI); + return githubEvents.sendGithubEvents(userInfo, streamInfo, process.env.APIURI); }) .catch(function(error){ - logger.logError(githubUsername, 'sync requested for unknown user', error); + logger.logError(githubUsername, 'error while trying to sync', error); }); res.status(200).send('ok, sync request acknowledged'); From 4a09a4ca0e1a01378bf15fff67d80153c62592c8 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Mon, 10 Aug 2015 09:16:52 +0100 Subject: [PATCH 53/66] added reauth unkonwn and allowed users without emails to log commits --- routes/githubEvents.js | 13 +++++++++---- views/reauthUnknownUser.html | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 views/reauthUnknownUser.html diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 6a56a97..c97dbf1 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -204,7 +204,7 @@ module.exports = function (mongoRepository, qdService) { var commitObjects = []; var userEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(userInfo.email); - var userEmail = userEmailParts[1] + userEmailParts[3]; + var userEmail = userEmailParts ? userEmailParts[1] + userEmailParts[3] : null; logger.logDebug(userInfo.githubUsername, 'flattening commits'); logger.logDebug(userInfo.githubUsername, 'removing push email alias, [original, unaliased]', [userInfo.email, userEmail]); @@ -218,9 +218,14 @@ module.exports = function (mongoRepository, qdService) { repo: event.repo.name } - var commitEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(commit.author.email); - var commitEmail = commitEmailParts[1] + commitEmailParts[3]; - commitReq.committerIsAuthor = commitEmail === userEmail; + commitReq.committerIsAuthor = true; + + if(userEmail){ + var commitEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(commit.author.email); + var commitEmail = commitEmailParts[1] + commitEmailParts[3]; + commitReq.committerIsAuthor = commitEmail === userEmail; + } + logger.logDebug(userInfo.githubUsername, 'commit req', commitReq); commitObjects.push(commitReq); }) diff --git a/views/reauthUnknownUser.html b/views/reauthUnknownUser.html new file mode 100644 index 0000000..ba58860 --- /dev/null +++ b/views/reauthUnknownUser.html @@ -0,0 +1,18 @@ + + + + + + + + 1self Github Integration + + +
+

Reauth: unknown user

+
+ + + + + \ No newline at end of file From ddb04b06ac4a39e8c094427e5848a8a568561f06 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Mon, 24 Aug 2015 15:01:39 +0100 Subject: [PATCH 54/66] changed the tag to hyphenated --- routes/githubEvents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index c97dbf1..da5afef 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -109,7 +109,7 @@ module.exports = function (mongoRepository, qdService) { 'github', 'computer', 'software', - 'source control' + 'source-control' ], 'dateTime': moment(event.commit.author.date).toISOString(), 'latestSyncField': { From a8862d049043cb3856592094de070e6371c45fa0 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Mon, 24 Aug 2015 15:12:56 +0100 Subject: [PATCH 55/66] changed push and commit tags --- routes/githubEvents.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index da5afef..8f7c09c 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -62,15 +62,15 @@ module.exports = function (mongoRepository, qdService) { if (event.type == 'PushEvent') { var singleEventTemplate = { 'actionTags': [ - 'Github', - 'Push' + 'push' ], 'source': 'GitHub', 'objectTags': [ - 'Computer', - 'Software', - 'Source', - 'Control' + 'git', + 'github', + 'computer', + 'software', + 'source-control' ], 'id': event.payload['push_id'], 'childIds': _.map(event.payload.commits, function (c) { From 9a9261c73fa2ac4e16d6391a5f1d42092c2b3cad Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Thu, 1 Oct 2015 17:05:22 +0100 Subject: [PATCH 56/66] improved logging --- routes/githubOAuth.js | 4 ++-- routes/qdService.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index d455b6c..df24718 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -101,12 +101,12 @@ module.exports = function (app, mongoRepository, oneselfService) { syncGithubEvents(callbackUrlForUser, stream.writeToken); res.redirect(req.session.appUri + "/integrations"); - }) + }); }, function (error) { res.render('error', { error: error }); - }) + }); }) .catch(function (error) { logError(userInfo.githubUsername, "Error in github callback: ", error); diff --git a/routes/qdService.js b/routes/qdService.js index 1f775d8..1ba658b 100644 --- a/routes/qdService.js +++ b/routes/qdService.js @@ -31,16 +31,18 @@ module.exports = function () { }; requestModule(options, function (e, response, body) { if (e) { + logger.error('error while registering stream: ', e); deferred.reject("Error: ", e); return; } if (response.statusCode === 401) { + logger.error('error while registering stream: ', [response.statusCode, response.body]); deferred.reject('auth error: check your appId and appSecret', null); return; } if (response.statusCode === 400) { - console.log(response.body); + logger.error('error while registering stream: ', [response.statusCode, response.body]); deferred.reject('Invalid username and registrationToken', null); return; } From de681b30d971572d550167b86fe6fd522d3fb752 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Sat, 3 Oct 2015 08:23:05 +0100 Subject: [PATCH 57/66] changed the name of the github integration database --- routes/githubOAuth.js | 1 + routes/mongoRepository.js | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index df24718..bc6619e 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -36,6 +36,7 @@ module.exports = function (app, mongoRepository, oneselfService) { req.session.githubUsername = githubUsername; var oneselfUsername = req.session.oneselfUsername; var registrationToken = req.session.registrationToken; + logDebug('reading user and registration token out of the session'); logDebug(githubUsername, "github User is: " + JSON.stringify(githubUser)); var callbackUrl = GITHUB_INT_CONTEXT_URI + '/authSuccess?username=' + githubUsername + '&latestSyncField={{latestSyncField}}' diff --git a/routes/mongoRepository.js b/routes/mongoRepository.js index 091f9a3..20d52e0 100644 --- a/routes/mongoRepository.js +++ b/routes/mongoRepository.js @@ -8,12 +8,14 @@ module.exports = function (mongoConnection) { console.log("connection object is : " + this.mongoConnection); }; + var userCollection = 'githubDatasourceUsers'; + this.findByGithubUsername = function (username) { var deferred = Q.defer(); var query = { githubUsername: username }; - mongoConnection.collection('users').findOne(query, function (error, user) { + mongoConnection.collection(userCollection).findOne(query, function (error, user) { if (error) { deferred.reject(error); } else { @@ -25,7 +27,7 @@ module.exports = function (mongoConnection) { this.insert = function (document) { var deferred = Q.defer(); - mongoConnection.collection('users').insert(document, function (error, numberOfDocsInserted) { + mongoConnection.collection(userCollection).insert(document, function (error, numberOfDocsInserted) { if (error) { deferred.reject(error); } else { @@ -37,7 +39,7 @@ module.exports = function (mongoConnection) { this.update = function (findQuery, updateQuery) { var deferred = Q.defer(); - mongoConnection.collection('users', function (err, collection) { + mongoConnection.collection(userCollection, function (err, collection) { collection.update(findQuery, updateQuery, function (error, data) { if (error) { deferred.reject(error); From f31cd1f29a62751b20f69be61334b7f718f42bd3 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 6 Oct 2015 10:54:01 +0100 Subject: [PATCH 58/66] added redirect uri --- app.js | 4 +++- routes/githubOAuth.js | 12 ++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app.js b/app.js index 457ed4b..8da1201 100644 --- a/app.js +++ b/app.js @@ -136,6 +136,7 @@ app.get('/', function (req, res) { process.env.appUri = req.session.appUri; req.session.oneselfUsername = req.query.username; req.session.registrationToken = req.query.token; + req.session.redirectUri = req.query.redirect_uri; logger.logInfo(req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); res.render('index'); }); @@ -159,9 +160,10 @@ app.get('/reauth', function (req, res) { app.get('/reauth/complete', function (req, res) { //req.session.integrationUri = req.headers['x-forwarded-host']; req.session.appUri = null; - req.session.redirect = null + req.session.redirect = null; req.session.reauth = null; req.session.username = null; + req.session.registrationToken = null; logger.logInfo(req.query.username, 'reauthing complete, ', [req.session.appUri, req.query.token]); logger.logDebug(req.query.username, 'serving reauth complete page'); res.render('reauthcomplete'); diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index bc6619e..aa18cc3 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -101,7 +101,7 @@ module.exports = function (app, mongoRepository, oneselfService) { .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); syncGithubEvents(callbackUrlForUser, stream.writeToken); - res.redirect(req.session.appUri + "/integrations"); + res.redirect(req.session.redirectUri); }); }, function (error) { res.render('error', { @@ -112,7 +112,7 @@ module.exports = function (app, mongoRepository, oneselfService) { .catch(function (error) { logError(userInfo.githubUsername, "Error in github callback: ", error); }); - } + }; var handleReauthCallback = function(req, res){ var githubUser = req.user.profile; @@ -158,24 +158,24 @@ module.exports = function (app, mongoRepository, oneselfService) { .then(function (user) { var query = { _id: user._id - } + }; var operation = { $set: { accessToken: userInfo.accessToken, email: userInfo.email } - } + }; mongoRepository.update(query, operation) .then(function () { res.redirect(req.session.redirect); - }) + }); }) .catch(function (error) { logError(userInfo.githubUsername, "Error in github callback: ", error); }); - } + }; var handleGithubCallback = function (req, res) { if(req.session.reauth){ handleReauthCallback(req, res); From e479a9770a2e3a0cb42186f09f07e9b4f9e32e8d Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Thu, 8 Oct 2015 20:21:45 +0100 Subject: [PATCH 59/66] added success=true redirect --- app.js | 5 +++++ routes/githubOAuth.js | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app.js b/app.js index 8da1201..fb3225a 100644 --- a/app.js +++ b/app.js @@ -8,6 +8,7 @@ var logger = require('morgan'); var bodyParser = require('body-parser'); var cookieParser = require('cookie-parser'); var mongoClient = require('mongodb').MongoClient; +var _ = require('underscore'); var GithubEvents = require('./routes/githubEvents'); var MongoRepository = require('./routes/mongoRepository'); @@ -15,6 +16,7 @@ var GithubOAuth = require('./routes/githubOAuth'); var QdService = require('./routes/qdService'); var app = express(); + /*jslint nomen: true */ app.use(express.static(path.join(__dirname, 'public'))); /*jslint nomen: false */ @@ -44,6 +46,9 @@ winston.info('PORT=' + process.env.PORT); var port = process.env.PORT || 5001; winston.info('APIURI=' + process.env.APIURI); +_.each(process.env, function(value, key){ + winston.info(key + '=' + value); +}); winston.info('SESSION_SECRET=' + process.env.SESSION_SECRET.substring(0,2) + '...'); var sessionSecret = process.env.SESSION_SECRET; diff --git a/routes/githubOAuth.js b/routes/githubOAuth.js index aa18cc3..cd34609 100644 --- a/routes/githubOAuth.js +++ b/routes/githubOAuth.js @@ -101,12 +101,12 @@ module.exports = function (app, mongoRepository, oneselfService) { .replace('{{latestSyncField}}', new Date(1970, 1, 1).toISOString()); syncGithubEvents(callbackUrlForUser, stream.writeToken); - res.redirect(req.session.redirectUri); + var redirectUri = req.session.redirectUri + '?success=true'; + res.redirect(redirectUri); }); }, function (error) { - res.render('error', { - error: error - }); + var redirectUri = req.session.redirectUri + '?success=false&error=server_error'; + res.redirect(redirectUri); }); }) .catch(function (error) { From 5ce67fa7108de163c5a96f74eb3caed509fa31d3 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Fri, 9 Oct 2015 09:47:56 +0100 Subject: [PATCH 60/66] removed sync github inter stitial --- app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.js b/app.js index fb3225a..c55cbc4 100644 --- a/app.js +++ b/app.js @@ -143,7 +143,7 @@ app.get('/', function (req, res) { req.session.registrationToken = req.query.token; req.session.redirectUri = req.query.redirect_uri; logger.logInfo(req.query.username, 'github setup started: appUri, registrationToken', [req.session.appUri, req.query.token]); - res.render('index'); + res.redirect('/auth/github'); }); app.get('/reauth', function (req, res) { From 3239e65510c235832bec96b212bb745a391f1ef6 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 29 Dec 2015 13:09:46 +0100 Subject: [PATCH 61/66] implemented sending of an error event when the github api cant auth with the token --- .jshintrc | 15 ++++ routes/githubEvents.js | 111 ++++++++++++++++++++++-------- routes/test/githubEvents.tests.js | 102 +++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 27 deletions(-) create mode 100644 .jshintrc create mode 100644 routes/test/githubEvents.tests.js diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000..2cb788b --- /dev/null +++ b/.jshintrc @@ -0,0 +1,15 @@ +{ + "boss": true, + "curly": true, + "eqeqeq": true, + "eqnull": true, + "immed": true, + "latedef": true, + "mocha" : true, + "newcap": true, + "noarg": true, + "node": true, + "sub": true, + "undef": true, + "unused": true +} diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 8f7c09c..3935768 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -8,6 +8,43 @@ var path = require('path'); module.exports = function (mongoRepository, qdService) { var logger; + var self = this; + + this.checkToken = function(userInfo, requestFunction, localLogger){ + var deferred = Q.defer(); + var options = { + url: "https://api.github.com/user/emails?access_token=" + userInfo.accessToken, + headers: { + "User-Agent": "1self" + } + }; + + requestFunction(options, function (err, res) { + if(err || res === undefined || res.statusCode !== 200){ + localLogger.logError(userInfo.githubUsername, JSON.stringify(res)); + localLogger.logError(userInfo.githubUsername, 'token is invalid'); + var result = { + message: err, + code: 500 + }; + + // it's likely when a sync error is seen that the github api is preventing token attacks + if(res.statusCode === 401){ + result.code = 401; + } + + localLogger.logError(userInfo.githubUsername, result); + deferred.reject(result); + } + else{ + localLogger.logInfo(userInfo.githubUsername, 'token is valid'); + deferred.resolve(res); + } + }); + return deferred.promise; + }; + + var getGithubPushEventsPerPage = function (page, userInfo) { @@ -15,7 +52,7 @@ module.exports = function (mongoRepository, qdService) { var user_api_url = '/users/' + githubUsername; var client = github.client(userInfo.accessToken); logger.logDebug(userInfo.githubUsername, 'fetching page [page, user_api_url, accessToken]', [page, user_api_url, userInfo.accessToken.substring(0,2)]); - client.get(user_api_url, {}, function (err, status, body, headers) { + client.get(user_api_url, {}, function () { }); var githubUser = client.user(githubUsername); @@ -56,15 +93,14 @@ module.exports = function (mongoRepository, qdService) { }; var convertEventsTo1SelfFormat = function (filteredEvents, username) { - var deferred = Q.defer(); + var convertEventTo1SelfFormat = function (acc, event) { logger.logSilly(username, 'converting raw event: ', event); - if (event.type == 'PushEvent') { + if (event.type === 'PushEvent') { var singleEventTemplate = { 'actionTags': [ 'push' ], - 'source': 'GitHub', 'objectTags': [ 'git', 'github', @@ -72,9 +108,9 @@ module.exports = function (mongoRepository, qdService) { 'software', 'source-control' ], - 'id': event.payload['push_id'], + 'id': event.payload.push_id, 'childIds': _.map(event.payload.commits, function (c) { - return c['sha'] + return c.sha; }), 'dateTime': moment(event.created_at).toISOString(), 'latestSyncField': { @@ -90,20 +126,19 @@ module.exports = function (mongoRepository, qdService) { return acc.concat(singleEventTemplate); } - else if (event.commit != undefined) { + else if (event.commit !== undefined) { var extensionStats = _.reduce(event.files, function(result, file){ var ext = path.extname(file.filename).substring(1); result[ext] = result[ext] || {}; - result[ext]['line-additions'] = (result[ext]['line-additions'] || 0) + file['additions']; - result[ext]['line-deletions'] = (result[ext]['line-deletions'] || 0) + file['deletions']; - result[ext]['line-changes'] = (result[ext]['line-changes'] || 0) + file['changes']; + result[ext]['line-additions'] = (result[ext]['line-additions'] || 0) + file.additions; + result[ext]['line-deletions'] = (result[ext]['line-deletions'] || 0) + file.deletions; + result[ext]['line-changes'] = (result[ext]['line-changes'] || 0) + file.changes; return result; }, {}); - var singleEventTemplate = { + var commitEventTemplate = { 'actionTags': [ 'commit' ], - 'source': 'GitHub', 'objectTags': [ 'git', 'github', @@ -132,19 +167,19 @@ module.exports = function (mongoRepository, qdService) { 'repo': event.repo, 'file-types': extensionStats } - } + }; if(event.committerIsAuthor === false){ - singleEventTemplate.actionTags = ['merge']; + commitEventTemplate.actionTags = ['merge']; } if (event.commit.author.email !== event.commit.committer.email) { - singleEventTemplate.actionTags = ['patch'] + commitEventTemplate.actionTags = ['patch']; } - logger.logSilly(username, 'converted commit event: ', singleEventTemplate); + logger.logSilly(username, 'converted commit event: ', commitEventTemplate); - return acc.concat(singleEventTemplate) + return acc.concat(commitEventTemplate); } else { logger.logSilly(username, 'couldnt convert as event is not a push or commit'); @@ -163,6 +198,7 @@ module.exports = function (mongoRepository, qdService) { if (_.isEmpty(events)) { logger.logDebug(userInfo.githubUsername, 'there are no events to send'); deferred.resolve(); + return; } logger.logDebug(userInfo.githubUsername, 'sending events to qd [event count, app uri]', [events.length, appUri]); @@ -182,17 +218,27 @@ module.exports = function (mongoRepository, qdService) { 'dateTime': moment().toISOString(), 'objectTags': ['1self', 'integration', 'sync'], 'actionTags': ['start'], - 'source': '1self-GitHub', 'properties': {} }; }; + var createSyncErrorEvent = function (code) { + return { + 'dateTime': moment().toISOString(), + 'objectTags': ['1self', 'integration', 'sync'], + 'actionTags': ['error'], + 'properties': { + code: code, + reauth: process.env.GITHUB_INT_CONTEXT_URI + '/reauth' + } + }; + }; + var createSyncCompleteEvent = function () { return { 'dateTime': moment().toISOString(), 'objectTags': ['1self', 'integration', 'sync'], 'actionTags': ['complete'], - 'source': '1self-GitHub', 'properties': {} }; }; @@ -213,10 +259,10 @@ module.exports = function (mongoRepository, qdService) { _.each(filteredEvents, function (event) { _.each(event.payload.commits, function (commit) { var commitReq = { - url: commit['url'], - pushId: event.payload['push_id'], + url: commit.url, + pushId: event.payload.push_id, repo: event.repo.name - } + }; commitReq.committerIsAuthor = true; @@ -228,7 +274,7 @@ module.exports = function (mongoRepository, qdService) { logger.logDebug(userInfo.githubUsername, 'commit req', commitReq); commitObjects.push(commitReq); - }) + }); }); logger.logDebug(userInfo.githubUsername, 'commit have been flattened, [commits]', commitObjects); @@ -274,7 +320,7 @@ module.exports = function (mongoRepository, qdService) { Q.all(promiseArray).then(function (commitEvents) { logger.logDebug(userInfo.githubUsername, 'all commit events retrieved'); - var events = filteredEvents.concat(commitEvents) + var events = filteredEvents.concat(commitEvents); deferred.resolve(events); }).catch(function (error) { logger.logError(userInfo.githubUsername, 'Error occurred :: getGithubCommitEvents', error); @@ -288,9 +334,14 @@ module.exports = function (mongoRepository, qdService) { var syncStartEvent = createSyncStartEvent(); qdService.sendEvent(syncStartEvent, streamInfo, appUri) + .then(function(){ + logger.logInfo(userInfo.githubUsername, 'testing token validity'); + logger.logInfo(userInfo.githubUsername, userInfo); + return self.checkToken(userInfo, request, logger); + }) .then(function () { logger.logInfo(userInfo.githubUsername, 'fetching events from github api', []); - return fetchGithubPushEvents(userInfo) + return fetchGithubPushEvents(userInfo); }) .then(function (events) { logger.logInfo(userInfo.githubUsername, 'filtering events to include only pushes, [event count]', [events.length]); @@ -318,12 +369,18 @@ module.exports = function (mongoRepository, qdService) { }) .catch(function (error) { logger.logError(userInfo.githubUsername, 'Error occurred :: sendGithubEvents', error); - }); + if(error.code === 401){ + logger.logError(userInfo.githubUsername, 'sending error event'); + var errorEvent = createSyncErrorEvent(error.code); + return qdService.sendEvent(errorEvent, streamInfo, appUri); + } + }) + .done() ; }; this.setLogger = function(newLogger){ logger = newLogger; - } + }; }; diff --git a/routes/test/githubEvents.tests.js b/routes/test/githubEvents.tests.js new file mode 100644 index 0000000..b44ddad --- /dev/null +++ b/routes/test/githubEvents.tests.js @@ -0,0 +1,102 @@ +'use strict'; + +var assert = require('assert'); +var GithubEvents = require('../githubEvents.js'); + +var events = new GithubEvents(null, null); + +var logger = { + messages:{ + verbose: [], + info: [], + warn: [], + debug: [], + silly: [] + }, + logDebug: function(message){ + this.messages.debug.push(message); + }, + logInfo: function(message){ + this.messages.info.push(message); + }, + logError: function(message){ + this.messages.info.push(message); + } +}; + +describe('githubEvents', function () { + it('check token rejects on any error', function () { + var userInfo = { + accessToken: 'at' + }; + + var request = function(options, callback){ + callback("error, unknown", {statusCode: 500}); + }; + + return events.checkToken(userInfo, request, logger) + .then(function(){ + assert(false, 'error wasnt rejected'); + }) + .catch(function(error){ + assert.equal(error.code, 500); + }); + }); + + it('check token translates 401 from github to 401 on our error', function () { + var userInfo = { + accessToken: 'at' + }; + + var request = function(options, callback){ + callback(null, {statusCode: 401}); + }; + + return events.checkToken(userInfo, request, logger) + .then(function(){ + assert(false, 'error wasnt rejected'); + }) + .catch(function(error){ + assert.equal(error.code, 401); + }); + }); + + it('check all other response errors are 500', function () { + var userInfo = { + accessToken: 'at' + }; + + var request = function(options, callback){ + // this response code doesn't mean anything, it's just not a 200 + callback(null, {statusCode: 480}); + }; + + return events.checkToken(userInfo, request, logger) + .then(function(){ + assert(false, 'error wasnt rejected'); + }) + .catch(function(error){ + assert.equal(error.code, 500); + }); + }); + + it('check 200 response gives a valid token', function () { + var userInfo = { + accessToken: 'at' + }; + + var request = function(options, callback){ + callback(null, {statusCode: 200}); + }; + + return events.checkToken(userInfo, request, logger) + .then(function(){ + assert(true, 'error wasnt rejected'); + }) + .catch(function(){ + assert(false, 'shouldnt get an error'); + }); + }); +}); + + From ba92525f73b80c6f8254fa7ef185b87c2054f26c Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 12 Jan 2016 11:57:22 +0000 Subject: [PATCH 62/66] added a check for the commit author conforming to the expected format --- routes/githubEvents.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 3935768..d0fa936 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -269,7 +269,9 @@ module.exports = function (mongoRepository, qdService) { if(userEmail){ var commitEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(commit.author.email); var commitEmail = commitEmailParts[1] + commitEmailParts[3]; - commitReq.committerIsAuthor = commitEmail === userEmail; + if(commitEmailParts.length === 4){ + commitReq.committerIsAuthor = commitEmail === userEmail; + } } logger.logDebug(userInfo.githubUsername, 'commit req', commitReq); From a61f0ade220fdd7ed88f1a16498d6b9a839a8a44 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 12 Jan 2016 12:50:52 +0000 Subject: [PATCH 63/66] moved email check into guard --- routes/githubEvents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index d0fa936..07cf0c0 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -268,8 +268,8 @@ module.exports = function (mongoRepository, qdService) { if(userEmail){ var commitEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(commit.author.email); - var commitEmail = commitEmailParts[1] + commitEmailParts[3]; if(commitEmailParts.length === 4){ + var commitEmail = commitEmailParts[1] + commitEmailParts[3]; commitReq.committerIsAuthor = commitEmail === userEmail; } } From 9f4f9b3b46bb68f620aedfbc266fca288d22682d Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 12 Jan 2016 12:52:45 +0000 Subject: [PATCH 64/66] added check for null --- routes/githubEvents.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index 07cf0c0..e394bfb 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -268,7 +268,7 @@ module.exports = function (mongoRepository, qdService) { if(userEmail){ var commitEmailParts = /(.*?)(\+.*?)?(@.*)/g.exec(commit.author.email); - if(commitEmailParts.length === 4){ + if(commitEmailParts && commitEmailParts.length === 4){ var commitEmail = commitEmailParts[1] + commitEmailParts[3]; commitReq.committerIsAuthor = commitEmail === userEmail; } From a2b98fd65eb0a243693fdf99071e8acb93e40cd3 Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 12 Jan 2016 14:15:50 +0000 Subject: [PATCH 65/66] when the commit events cant be retrieved, treat it as an error --- routes/githubEvents.js | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index e394bfb..ae9507f 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -326,6 +326,7 @@ module.exports = function (mongoRepository, qdService) { deferred.resolve(events); }).catch(function (error) { logger.logError(userInfo.githubUsername, 'Error occurred :: getGithubCommitEvents', error); + deferred.reject(error); }); return deferred.promise; From e30aae17bac4f90ae99d356aa3d60a34fa0f09aa Mon Sep 17 00:00:00 2001 From: Ed Sykes Date: Tue, 12 Jan 2016 16:39:55 +0000 Subject: [PATCH 66/66] removed check for error code before sending the error event --- routes/githubEvents.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/routes/githubEvents.js b/routes/githubEvents.js index ae9507f..9d028bc 100644 --- a/routes/githubEvents.js +++ b/routes/githubEvents.js @@ -372,11 +372,9 @@ module.exports = function (mongoRepository, qdService) { }) .catch(function (error) { logger.logError(userInfo.githubUsername, 'Error occurred :: sendGithubEvents', error); - if(error.code === 401){ - logger.logError(userInfo.githubUsername, 'sending error event'); - var errorEvent = createSyncErrorEvent(error.code); - return qdService.sendEvent(errorEvent, streamInfo, appUri); - } + logger.logError(userInfo.githubUsername, 'sending error event'); + var errorEvent = createSyncErrorEvent(error.code); + return qdService.sendEvent(errorEvent, streamInfo, appUri); }) .done() ; };