Skip to content

Commit

Permalink
Implement OpenUserJS#81 - Sync Script Description from GitHub
Browse files Browse the repository at this point in the history
* Unfocused md-editors no longer look disabled.
* Add new panel in the scriptEditMetadataPage to with inputs to enter the filepath to a markdown file to use as the script description.
* Refactored the webhook to wait on saving the script/description before sending an OK response with a list of the models changed (or an error).
* Refactored the github import code to also handle importing markdown files.
* New Properties:
    * Script.githubSyncDescription
    * Script.githubSyncSource
    * Script.githubSyncUserId
    * Script.githubSyncRepoName
    * Script.githubSyncDescriptionPath
    * Script.githubSyncSourcePath
* Automatically fill out githubSyncUserId, githubSyncRepoName, and githubSyncSourcePath when when importing from github (eg: during the webhook).
* Disable the Script.about markdown editor when set to sync with github.
  • Loading branch information
Zren committed Dec 7, 2014
1 parent 8b27fa1 commit 346137c
Show file tree
Hide file tree
Showing 17 changed files with 703 additions and 324 deletions.
113 changes: 113 additions & 0 deletions controllers/githubHook.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
var async = require('async');
var _ = require('underscore');

var User = require('../models/user').User;

var github = require('../libs/githubClient');
var githubImporter = require('../libs/githubImporter');

// GitHub calls this on a push if a webhook is setup
// This controller makes sure we have the latest version of a script
module.exports = function (aReq, aRes, aNext) {
if (!aReq.body.payload)
return aRes.status(400).send('Payload required.');

if (process.env.NODE_ENV === 'production') {
// Test for know GH webhook ips: https://api.github.com/meta
var reqIP = aReq.headers['x-forwarded-for'] || aReq.connection.remoteAddress;
if (!/192\.30\.25[2-5]\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])$/.test(reqIP))
return;
}


var payload = JSON.parse(aReq.body.payload);

// Only accept commits to the master branch
if (!payload || payload.ref !== 'refs/heads/master')
return aRes.send(400, 'payload.ref !== refs/heads/master');

var githubUserName = payload.repository.owner.name;
var githubRepoName = payload.repository.name;


User.findOne({
ghUsername: githubUserName
}, function (aErr, aUser) {
if (!aUser)
return aRes.status(400).send('No account linked to GitHub username ' + username);

// Gather the modified user scripts
var jsFilenames = {}; // Set (key == value)
var mdFilenames = {}; // Set (key == value)
payload.commits.forEach(function (aCommit) {
aCommit.modified.forEach(function (aFilename) {
switch (aFilename.substr(-3)) {
case '.js':
jsFilenames[aFilename] = aFilename;
break;
case '.md':
mdFilenames[aFilename] = aFilename;
break;
}
});
});
jsFilenames = Object.keys(jsFilenames);
mdFilenames = Object.keys(mdFilenames);

var githubRepoBlobs = null;

// Update
async.series([
// Fetch all blobs in the target repo.
function(aCallback) {
async.waterfall([
function(aCallback) {
github.gitdata.getBlobs({
user: encodeURIComponent(githubUserName),
repo: encodeURIComponent(githubRepoName)
}, aCallback);
},

function(aBlobs, aCallback) {
githubRepoBlobs = aBlobs;
aCallback();
},
], aCallback);
},

// Update Javascript File Triggers
function(aCallback) {
async.map(jsFilenames, function(jsFilename, aCallback) {
githubImporter.importJavascriptBlob({
user: aUser,
githubUserId: githubUserName,
githubRepoName: githubRepoName,
githubBlobPath: jsFilename,
updateOnly: false,
blobs: githubRepoBlobs
}, aCallback);
}, aCallback);
},

// Update Markdown File Triggers
function(aCallback) {
async.map(mdFilenames, function(mdFilename, aCallback) {
githubImporter.importMarkdownBlob({
user: aUser,
githubUserId: githubUserName,
githubRepoName: githubRepoName,
githubBlobPath: mdFilename,
blobs: githubRepoBlobs
}, aCallback);
}, aCallback);
}
], function(aError, aResults) {
if (aError) {
console.error(aError);
return aRes.status(500).send('Error while updating.');
}

aRes.status(200).send(aResults);
});
});
};
50 changes: 50 additions & 0 deletions controllers/githubImport.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
var githubImporter = require('../libs/githubImporter');
var modelParser = require('../libs/modelParser');
var statusCodePage = require('../libs/templateHelpers').statusCodePage;

module.exports = function (aReq, aRes, aNext) {
var authedUser = aReq.session.user;

if (!authedUser) return aRes.redirect('/login');

//
var options = {};
var tasks = [];

// Session
authedUser = options.authedUser = modelParser.parseUser(authedUser);
options.isMod = authedUser && authedUser.isMod;
options.isAdmin = authedUser && authedUser.isAdmin;

// GitHub
var githubUserId = options.githubUserId = aReq.body.user || aReq.query.user || authedUser.ghUsername || authedUser.githubUserId();
var githubRepoName = options.githubRepoName = aReq.body.repo || aReq.query.repo;
var githubBlobPath = options.githubBlobPath = aReq.body.path || aReq.query.path;

if (!(githubUserId && githubRepoName && githubBlobPath)) {
return statusCodePage(aReq, aRes, aNext, {
statusCode: 400,
statusMessage: 'Bad Request. Require <code>user</code>, <code>repo</code>, and <code>path</code> to be set.'
});
}

githubImporter.importJavascriptBlob({
user: authedUser,
githubUserId: githubUserId,
githubRepoName: githubRepoName,
githubBlobPath: githubBlobPath,
updateOnly: false
}, function (aErr, aData) {
if (aErr) {
console.error(aErr);
console.error(githubUserId, githubRepoName, githubBlobPath);
return statusCodePage(aReq, aRes, aNext, {
statusCode: 400,
statusMessage: aErr
});
}

var script = modelParser.parseScript(aData.script);
aRes.redirect(script.scriptPageUrl);
});
};
43 changes: 42 additions & 1 deletion controllers/script.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ var modelQuery = require('../libs/modelQuery');
var modelParser = require('../libs/modelParser');
var countTask = require('../libs/tasks').countTask;
var pageMetadata = require('../libs/templateHelpers').pageMetadata;
var statusCodePage = require('../libs/templateHelpers').statusCodePage;
var githubImporter = require('../libs/githubImporter');

// Let controllers know this is a `new` route
exports.new = function (aController) {
Expand Down Expand Up @@ -398,10 +400,49 @@ exports.edit = function (aReq, aRes, aNext) {
});
} else if (typeof aReq.body.about !== 'undefined') {
// POST

aScriptData.about = aReq.body.about;

var wasSyncingAbout = aScriptData.githubSyncDescription;

aScriptData.githubSyncUserId = aReq.body.githubSyncUserId;
aScriptData.githubSyncRepoName = aReq.body.githubSyncRepoName;
aScriptData.githubSyncDescriptionPath = aReq.body.githubSyncDescriptionPath;

aScriptData.githubSyncDescription = aReq.body.githubSyncDescription;

// Validate that all fields are filled out.
if (!(aScriptData.githubSyncUserId && aScriptData.githubSyncRepoName && aScriptData.githubSyncDescriptionPath))
aScriptData.githubSyncDescription = false;

var scriptGroups = (aReq.body.groups || "");
scriptGroups = scriptGroups.split(/,/);
addScriptToGroups(aScriptData, scriptGroups, function () {

// The following function will save the Script even if scriptGroups is empty.
async.series([
function(aCallback) {
addScriptToGroups(aScriptData, scriptGroups, aCallback);
},
function(aCallback) {
// If we just entered the sync information, pull it now.
if (!wasSyncingAbout && aScriptData.githubSyncDescription) {
githubImporter.importMarkdownBlob({
user: authedUser,
githubUserId: aScriptData.githubSyncUserId,
githubRepoName: aScriptData.githubSyncRepoName,
githubBlobPath: aScriptData.githubSyncDescriptionPath
}, aCallback);
} else {
aCallback();
}
},
], function(aErr) {
if (aErr) {
return statusCodePage(aReq, aRes, aNext, {
statusCode: 500,
statusMessage: 'Error while updating script metadata.'
});
}
aRes.redirect(script.scriptPageUrl);
});
} else {
Expand Down
163 changes: 0 additions & 163 deletions controllers/scriptStorage.js
Original file line number Diff line number Diff line change
Expand Up @@ -132,120 +132,6 @@ exports.sendMeta = function (aReq, aRes, aNext) {
});
};

// Modified from Count Issues (http://userscripts.org/scripts/show/69307)
// By Marti Martz (http://userscripts.org/users/37004)
function parseMeta(aString, aNormalize) {
var rLine = /\/\/ @(\S+)(?:\s+(.*))?/;
var headers = {};
var name = null;
var prefix = null;
var key = null;
var value = null;
var line = null;
var lineMatches = null;
var lines = {};
var uniques = {
'description': true,
'icon': true,
'name': true,
'namespace': true,
'version': true,
'oujs:author': true
};
var unique = null;
var one = null;
var matches = null;

lines = aString.split(/[\r\n]+/).filter(function (aElement, aIndex, aArray) {
return (aElement.match(rLine));
});

for (line in lines) {
var header = null;

lineMatches = lines[line].replace(/\s+$/, '').match(rLine);
name = lineMatches[1];
value = lineMatches[2];
if (aNormalize) {
// Upmix from...
switch (name) {
case 'homepage':
case 'source':
case 'website':
name = 'homepageURL';
break;
case 'defaulticon':
case 'iconURL':
name = 'icon';
break;
case 'licence':
name = 'license';
break;
}
}
name = name.split(/:/).reverse();
key = name[0];
prefix = name[1];
if (key) {
unique = {};
if (prefix) {
if (!headers[prefix]) {
headers[prefix] = {};
}
header = headers[prefix];
if (aNormalize) {
for (one in uniques) {
matches = one.match(/(.*):(.*)$/);
if (uniques[one] && matches && matches[1] === prefix) {
unique[matches[2]] = true;
}
}
}
} else {
header = headers;
if (aNormalize) {
for (one in uniques) {
if (uniques[one] && !/:/.test(one)) {
unique[one] = true;
}
}
}
}
if (!header[key] || aNormalize && unique[key]) {
header[key] = value || '';
} else if (!aNormalize || header[key] !== (value || '')
&& !(header[key] instanceof Array && header[key].indexOf(value) > -1)) {
if (!(header[key] instanceof Array)) {
header[key] = [header[key]];
}
header[key].push(value || '');
}
}
}
return headers;
}
exports.parseMeta = parseMeta;

exports.getMeta = function (aChunks, aCallback) {
// We need to convert the array of buffers to a string to
// parse the header. But strings are memory inefficient compared
// to buffers so we only convert the least number of chunks to
// get the user script header.
var str = '';
var i = 0;
var header = null;

for (; i < aChunks.length; ++i) {
header = null;
str += aChunks[i];
header = /^(?:\uFEFF)?\/\/ ==UserScript==([\s\S]*?)^\/\/ ==\/UserScript==/m.exec(str);

if (header && header[1]) { return aCallback(parseMeta(header[1], true)); }
}

aCallback(null);
};

exports.storeScript = function (aUser, aMeta, aBuf, aCallback, aUpdate) {
var s3 = new AWS.S3();
var scriptName = null;
Expand Down Expand Up @@ -380,52 +266,3 @@ exports.deleteScript = function (aInstallName, aCallback) {
});
});
};

// GitHub calls this on a push if a webhook is setup
// This controller makes sure we have the latest version of a script
exports.webhook = function (aReq, aRes) {
var RepoManager = require('../libs/repoManager');
var payload = null;
var username = null;
var reponame = null;
var repos = {};
var repo = null;

aRes.end(); // Close connection

// Test for know GH webhook ips: https://api.github.com/meta
if (!aReq.body.payload ||
!/192\.30\.25[2-5]\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])$/
.test(aReq.headers['x-forwarded-for'] || aReq.connection.remoteAddress)) {
return;
}

payload = JSON.parse(aReq.body.payload);

// Only accept commits to the master branch
if (!payload || payload.ref !== 'refs/heads/master') { return; }

// Gather all the info for the RepoManager
username = payload.repository.owner.name;
reponame = payload.repository.name;

repo = repos[reponame] = {};

// Find the user that corresponds the repo owner
User.findOne({ ghUsername: username }, function (aErr, aUser) {
if (!aUser) { return; }

// Gather the modified user scripts
payload.commits.forEach(function (aCommit) {
aCommit.modified.forEach(function (aFilename) {
if (aFilename.substr(-8) === '.user.js') {
repo[aFilename] = '/' + encodeURI(aFilename);
}
});
});

// Update modified scripts
var repoManager = RepoManager.getManager(null, aUser, repos);
repoManager.loadScripts(function () { }, true);
});
};
Loading

0 comments on commit 346137c

Please sign in to comment.