save point durring huge unorganized refactor

This commit is contained in:
Spencer Alger
2013-11-22 16:48:30 -07:00
parent 5bb70fbe58
commit 97ba084795
80 changed files with 46126 additions and 2410 deletions

View File

@ -95,9 +95,13 @@ function transformFile(entry) {
});
});
spec.urls = _.map(_.sortBy(urls, 'sortOrder'), function (url) {
return _.omit(url, 'sortOrder');
});
if (urls.length > 1) {
spec.urls = _.map(_.sortBy(urls, 'sortOrder'), function (url) {
return _.omit(url, 'sortOrder');
});
} else {
spec.url = urls[0];
}
spec.params = _.transform(spec.params, function (note, param, name) {
// param.name = name;
@ -106,6 +110,10 @@ function transformFile(entry) {
]);
}, {});
if (_.size(spec.params) === 0) {
delete spec.params;
}
// escape method names with "special" keywords
var location = spec.name.split('.').join('.prototype.')
.replace(/(^|\.)(delete|default)(\.|$)/g, '[\'$2\']');
@ -114,6 +122,7 @@ function transformFile(entry) {
spec: _.pick(spec, [
'methods',
'params',
'url',
'urls',
'needBody',
'bulkBody',
@ -126,6 +135,61 @@ function transformFile(entry) {
allParams: allParams
};
function hasMethod(/* ...methods */) {
for (var i = 0; i < arguments.length; i++) {
if (~action.spec.methods.indexOf(arguments[i])) {
continue;
} else {
return false;
}
}
return true;
}
function methodsAre(/* ...methods */) {
return hasMethod.apply(null, arguments) && arguments.length === action.spec.methods.length;
}
var method;
if (action.spec.methods.length === 1) {
method = action.spec.methods[0];
} else {
// we need to define what the default method(s) will be
if (hasMethod('DELETE', 'POST')) {
method = 'POST';
}
else if (methodsAre('DELETE')) {
method = 'DELETE';
}
else if (methodsAre('POST', 'PUT')) {
method = 'POST';
}
else if (methodsAre('GET', 'POST')) {
method = 'POST';
}
else if (methodsAre('GET', 'HEAD')) {
if (action.spec.castExists) {
method = 'HEAD';
} else {
method = 'GET';
}
}
}
if (method) {
if (method !== 'GET') {
action.spec.method = method;
}
delete action.spec.methods;
} else {
throw new Error('unable to pick a method for ' + JSON.stringify(action, null, ' '));
}
if (action.name === 'create') {
action.proxy = 'index';
action.transformBody = 'params.op_type = \'create\';';
}
if (actions.push(action) === specCount && doneParsing) {
module.exports.emit('ready', action);
}

View File

@ -17,12 +17,21 @@ function download() {
}
}));
// seperate the proxy actions
var groups = _.groupBy(actions, function (action) {
return action.proxy ? 'proxies' : 'normal';
});
clean(outputPath);
console.log('writing', actions.length, 'api actions to', outputPath);
fs.writeFileSync(outputPath, templates.apiFile({
actions: actions,
actions: groups.normal,
proxies: groups.proxies,
namespaces: _.unique(namespaces.sort(), true)
}));
fs.writeFileSync(docOutputPath, templates.apiDocs({
actions: actions
}));
@ -30,7 +39,7 @@ function download() {
}
restSpecUpdated(function (err, updated) {
if (process.env.FORCE_GEN || err || updated) {
if (process.env.FORCE_GEN || process.env.npm_config_force || err || updated) {
download();
}
});

View File

@ -1,73 +0,0 @@
var _ = require('<%= path2lib %>utils');
var errors = require('<%= path2lib %>errors');<%
if (_.keys(enumOptions).length) {
%>
<% _.each(enumOptions, function(options, name) {
%>
var <%= name %>Options = <%= stringify(options) %>;<%
});
}
%>
/**
* Perform an elasticsearch [<%= name %>](<%= docUrl %>) request
*
* @for Client
* @method <%= name %>
* @param {Object} params - An object with parameters used to carry out this action<% _.each(params, function(param, paramName) { %>
* @param {<%= paramType(param.type) %>} <%= paramWithDefault('params.' + paramName, param.default) %><% if (param.description) { %> - <%= param.description %><% } %><%
})
%>
*/
function do<%= _.studlyCase(name) %>(params, cb) {
if (typeof params === 'function') {
cb = params;
params = {};
} else {
params = params || {};
cb = typeof cb === 'function' ? cb : _.noop;
}
var request = {
<%= writeRequestObjectBody(4, name, body, methods) %>
};
var parts = {};
var query = {};
var responseOpts = {};
<%
if (methods.length > 1) { %>
// figure out the method
if (params.method = _.toUpperString(params.method)) {
if (<%= _.map(methods, function (method) { return 'params.method === ' + stringify(method) }).join(' || ') %>) {
request.method = params.method;
} else {
throw new TypeError('Invalid method: should be one of <%= methods.join(', ') %>');
}
} else {<%
if (_.contains(methods, 'GET')) {
var nonGet = _.find(methods, function (m) {return m !== 'GET'; });%>
request.method = params.body ? <%= stringify(nonGet) %> : 'GET';<%
} else {%>
request.method = <%= stringify(methods[0]) %>;<%
}%>
}<%
}
%>
// find the paths's params
<%= writeParams(2, urlParts, 'parts.') %>
// build the path
<%= writeUrls(2, urls, urlParts, params) %>
// build the query string
<%= writeParams(2, params, 'query.') %>
request.path = request.path + _.makeQueryString(query);
<%= returnStatement(2, name) %>
}
module.exports = do<%= _.studlyCase(name) %>;

View File

@ -12,16 +12,16 @@ _.each(actions, function (action) {
var className = _.studlyCase(namespace) + 'NS';
%>
api.<%= namespace %> = function <%= className %>(client) {
if (this instanceof <%= className %>) {
this.client = client;
} else {
return new <%= className %>(client);
}
api.<%= namespace %> = function <%= className %>(transport) {
this.transport = transport;
};<%
}
%>
}%>
<%= partials.client_action(action) %><%
}); %>
});
_.each(proxies, function (action) {%>
<%= partials.client_action_proxy(action) %><%
});
%>

View File

@ -0,0 +1,19 @@
/**
* Perform a [<%= name %>](<%= docUrl %>) request
*
* @param {Object} params - An object with parameters used to carry out this action<%
_.each(allParams, function(param, paramName) { %>
* @param {<%= paramType(param.type) %>} <%= paramWithDefault('params.' + paramName, param.default) %><%
if (param.description) {
%> - <%= param.description %><%
}
%><% }) %>
*/
api<%= (location[0] === '[' ? '' : '.') + location %> = ca.proxy(<%= 'api' + (proxy[0] === '[' ? '' : '.') + proxy %><%
if (typeof transformBody === 'string') { %>, {
transform: function (params) {
<%= indent(transformBody, 4) %>
}
}<%
}
%>);

View File

@ -42,6 +42,13 @@ var templateGlobals = {
_: _,
indent: function (block, spaces) {
var indent = _.repeat(' ', spaces);
return block.split('\n').map(function (line) {
return indent + line;
}).join('\n');
},
paramType: function (type) {
switch (type && type.toLowerCase ? type.toLowerCase() : 'any') {
case 'time':

View File

@ -34,7 +34,8 @@ var es = require('../../../src/elasticsearch'),
endingMoment = moment().endOf('day').add('days', days),
clientConfig = {
log: {
level: 'error'
level: 'trace',
type: 'stdio'
}
};
@ -95,7 +96,7 @@ fillIndecies(function () {
actions.push(event);
if (actions.length === 3000 || i === count - 1) {
client.config.log.info('writing', actions.length / 2, 'documents');
console.info('writing', actions.length / 2, 'documents');
client.bulk({
body: actions
}, done);
@ -176,10 +177,11 @@ function fillIndecies(cb) {
async.parallel(indexPushActions, function (err, responses) {
if (err) {
client.config.log.error(err.message = 'Unable to create indicies: ' + err.message);
console.error(err.message = 'Unable to create indicies: ' + err.message);
console.error(err.stack);
} else {
_.each(_.groupBy(responses), function (list, did) {
client.config.log.info(list.length, 'indicies', did);
console.info(list.length, 'indicies', did);
});
cb();
}

View File

@ -32,7 +32,7 @@ function download() {
restSpecUpdated(function (err, updated) {
if (process.env.FORCE_GEN || err || updated) {
if (process.env.FORCE_GEN || process.env.npm_config_force || err || updated) {
download();
}
});

View File

@ -0,0 +1,81 @@
/**
* The JUnit xml output desired by Jenkins essentially looks like this:
*
* testsuites:
* - testsuite: (name, timestamp, hostname, tests, failures, errors, time)
* - testcase: (error or failure, name, classname, time)
*
* Full XSD avaliable [here](http://windyroad.com.au/dl/Open%20Source/JUnit.xsd)
*
* from
*
* {
* stats: {
*
* }
* suite: [
* {
* name:
* results: []
* suites: [] // optional
* }
* ]
* }
*/
module.exports = makeJUnitXml;
var testXml = require('xmlbuilder');
var suites = testXml.create('testsuites');
var suiteCount = 0;
var moment = require('moment');
var _ = require('lodash');
function makeJUnitXml(runnerName, testDetails) {
_.each(testDetails.suites, function serializeSuite(suiteInfo) {
var suite = suites.ele('testsuite', {
package: 'elasticsearch-js:yaml_tests',
id: suiteCount++,
name: suiteInfo.name,
timestamp: moment(suiteInfo.start).toJSON(),
hostname: 'localhost',
tests: (suiteInfo.results && suiteInfo.results.length) || 0,
failures: _.where(suiteInfo.results, {pass: false}).length,
errors: 0,
time: suiteInfo.time / 1000
});
_.each(suiteInfo.results, function (testInfo) {
var parts = suiteInfo.name.replace(/\.yaml$/, '').replace(/\./g, '_').split(/\//);
var section = parts.shift();
var behavior = parts.join('/');
var testcase = suite.ele('testcase', {
name: behavior + ' - ' + testInfo.name,
time: (testInfo.time || 0) / 1000,
classname: runnerName + '.' + section
});
if (testInfo.errMsg) {
testcase.ele('failure', {
message: testInfo.errMsg,
type: 'AssertError'
});
} else if (!testInfo.pass) {
testcase.ele('error', {
message: 'Unknown Error',
type: 'TestError'
});
}
});
if (suiteInfo.suites) {
_.each(suiteInfo.suites, serializeSuite);
}
suite.ele('system-out', {}).cdata(suiteInfo.stdout);
suite.ele('system-err', {}).cdata(suiteInfo.stderr);
});
return suites.toString({ pretty: true});
}

View File

@ -1,5 +1,11 @@
var https = require('https');
var lastCommitUrl = 'https://api.github.com/repos/elasticsearch/elasticsearch-rest-api-spec/commits/HEAD';
var request = {
hostname: 'api.github.com',
path: '/repos/elasticsearch/elasticsearch-rest-api-spec/commits/HEAD',
headers: {
'User-Agent': 'spenceralger'
}
};
var fs = require('fs');
var lastRestSpecUpdateFile = __dirname + '/last_rest_spec_update.sha';
@ -10,7 +16,7 @@ if (fs.existsSync(lastRestSpecUpdateFile)) {
lastRestSpecUpdate = fs.readFileSync(lastRestSpecUpdateFile, 'utf8');
}
var req = https.get(lastCommitUrl, function (incoming) {
var req = https.get(request, function (incoming) {
if (incoming.statusCode !== 200) {
req.abort();
console.error('request for last commit failed', incoming.statusCode, incoming.headers);

View File

@ -0,0 +1,146 @@
var server = require('./server');
var child_process = require('child_process');
var _ = require('lodash');
var open = require('open');
var fs = require('fs');
var path = require('path');
var async = require('async');
var yamlTestSourceFile = path.join(__dirname, '../../test/integration/yaml_suite/index.js');
var yamlTestBundleFile = path.join(__dirname, '../../test/browser_integration/yaml_tests.js');
var clientEntryFile = path.join(__dirname, '../../src/elasticsearch.js');
var browsers = _.transform({
safari: {
darwin: 'Safari'
},
chrome: {
darwin: 'Google Chrome',
win32: 'Google Chrome',
executable: 'google-chrome'
},
chromium: {
executable: 'chromium-browser',
},
firefox: {
darwin: 'Firefox',
win32: 'Firefox',
executable: 'firefox'
},
opera: {
darwin: 'Opera',
win32: 'Opera',
executable: 'opera'
}
}, function (browsers, config, name) {
if (config[process.platform]) {
browsers[name] = config[process.platform];
return;
}
if (process.platform !== 'darwin' && process.platform !== 'win32' && config.executable) {
browsers[name] = config.executable;
return;
}
}, {});
var argv = require('optimist')
.default('browser', 'chrome')
.default('force_gen', false)
.boolean('force_gen')
.alias('f', 'force_gen')
.default('host', 'localhost')
.default('port', 9200)
.argv;
var browserAppName;
async.series([
function (done) {
if (browsers.hasOwnProperty(argv.browser)) {
browserAppName = browsers[argv.browser];
done();
} else {
done('--browser must be set to one of ' + _.keys(browsers).join(', ') + ' on this platform');
}
},
function (done) {
fs.exists('dist', function (yes) {
if (!argv.force_gen && yes) {
done();
return;
}
console.log('generating client with "grunt build"');
child_process.spawn('grunt', ['build'], {
stdio: 'inherit'
}).on('close', function (status) {
done(status && 'grunt closed with a status code of ' + status + '. aborting.');
});
});
},
function (done) {
fs.exists(yamlTestBundleFile, function (yes) {
if (!argv.force_gen && yes) {
done();
return;
}
console.log('generating browser\'s yaml_tests.js bundle');
var b = require('browserify')();
b.add(yamlTestSourceFile);
var bundle = b.bundle({
external: [
'optimist'
],
ignore: [
'test/integration/yaml_suite/reporter',
clientEntryFile
]
});
var file = fs.createWriteStream(yamlTestBundleFile, {
flags: 'w',
encoding: 'utf8',
mode: 0666
});
bundle.pipe(file);
file.once('error', function (err) {
done(err);
});
bundle.once('error', function (err) {
done(err);
});
bundle.once('end', function () {
done();
});
});
}
], function (err) {
if (err) {
console.error(err);
process.exit(1);
} else {
server.listen(0, function () {
var port = server.address().port;
console.log('server listening on port', port);
open('http://localhost:' + port + '?es_hostname=' + encodeURIComponent(argv.host) +
'&es_port=' + encodeURIComponent(argv.port) +
'&browser=' + encodeURIComponent(argv.browser), browserAppName);
});
server.on('tests done', function (success) {
console.log('test completed', success ? 'successfully' : 'but failed');
process.exit(success ? 0 : 1);
});
}
});

View File

@ -0,0 +1,190 @@
var http = require('http');
var url = require('url');
var path = require('path');
var fs = require('fs');
var _ = require('lodash');
var chalk = require('chalk');
var makeJUnitXml = require('../make_j_unit_xml');
chalk.enabled = true;
var middleware = [];
Error.stackTraceLimit = Infinity;
var chars = 'abcdefghijklmnopqrstuvwxyz';
var server = http.createServer(function (req, resp) {
var parsedUrl = url.parse(req.url, true);
req.uri = parsedUrl.pathname;
req.query = parsedUrl.query;
req.filename = path.join(__dirname, '../../test/browser_integration/', req.uri);
var end = resp.end;
resp.end = function () {
console.log(chalk[this.statusCode < 300 ? 'green' : 'red'](this.statusCode), req.uri);
end.apply(resp, arguments);
};
var middleIndex = -1;
function next() {
middleIndex++;
if (middleIndex < middleware.length) {
middleware[middleIndex](req, resp, next);
} else {
resp.writeHead(500);
resp.end('500 Bad Gateway\n');
}
}
next();
});
function rand(length) {
var str = '';
while (str.length < length) {
str += chars.charAt(Math.floor(Math.random() * chars.length));
}
return str;
}
function collectTestResults(req, resp) {
var body = '';
var browser = req.query.browser;
var logFilename = path.join(__dirname, '../../test-output-' + browser + '.xml');
req.on('data', function (chunk) {
body += chunk;
});
req.on('error', function (err) {
resp.writeHead(500);
resp.end(err.message || 'failed to receive request completely');
});
req.on('end', function () {
var testDetails;
try {
testDetails = JSON.parse(body);
} catch (e) {
resp.writeHead(500);
resp.end('encoding failure');
return;
}
resp.writeHead(200);
resp.end('good work');
var xml = makeJUnitXml(browser, testDetails);
fs.writeFile(logFilename, xml, function (err) {
if (err) {
console.log('unable to save test-output to', err.message);
console.trace();
server.emit('tests done', false);
} else {
console.log('test output written to', logFilename);
server.emit('tests done', !testDetails.stats.failures);
}
});
});
}
middleware.push(function (req, resp, next) {
// resolve filenames
switch (req.uri) {
case '/tests-started':
resp.end('OK');
return;
case '/tests-complete':
return collectTestResults(req, resp);
case '/expect.js':
req.filename = path.join(__dirname, '../../node_modules/expect.js/expect.js');
break;
case '/mocha.js':
case '/mocha.css':
req.filename = path.join(__dirname, '../../node_modules/mocha', req.uri);
break;
case '/client.js':
req.filename = path.join(__dirname, '../../dist/elasticsearch.js');
break;
}
next();
});
middleware.push(function (req, resp, next) {
// catch 404's, add directory's index.html
fs.stat(req.filename, function (err, stats) {
if (err) {
resp.writeHead(404, {'Content-Type': 'text/plain'});
resp.write('404 Not Found\n');
resp.end();
} else {
if (stats.isDirectory()) {
req.filename = path.join(req.filename, '../../test/browser_integration/index.html');
}
next();
}
});
});
middleware.push(function (req, resp) {
// static files
var reader = fs.createReadStream(req.filename);
var data = '';
reader.on('data', onData);
reader.on('error', onError);
reader.on('end', onEnd);
function cleanupListeners() {
reader.removeListener('end', onEnd);
reader.removeListener('data', onData);
reader.removeListener('error', onError);
}
function onData(chunk) {
data += chunk;
}
function onError(err) {
cleanupListeners();
console.error(err);
resp.setHeader('Content-Type', 'text/plain');
resp.writeHead(500);
resp.write(err.message + '\n');
resp.end();
}
function onEnd() {
cleanupListeners();
var contentType = 'text/plain';
switch (req.filename.split('.').pop()) {
case 'js':
contentType = 'application/javascript';
break;
case 'css':
contentType = 'text/css';
break;
case 'html':
contentType = 'text/html';
break;
}
if (contentType === 'text/html') {
resp.end(_.template(data, _.defaults(req.query, {
es_hostname: 'localhost',
es_port: 9200,
browser: 'unknown',
ts: 'no'//rand(5)
})));
} else {
resp.end(data);
}
}
});
module.exports = server;

81
scripts/run_tests.js Normal file
View File

@ -0,0 +1,81 @@
var async = require('async');
var cp = require('child_process');
var chalk = require('chalk');
var argv = require('optimist')
.default({
'check-upstream': false,
'in-node': true,
'in-browser': true,
'not-in-node': false,
'not-in-browser': false,
'unit': true,
'integration': true
})
.alias({
u: 'unit',
i: 'integration',
b: 'in-browser',
n: 'in-node',
})
.parse(JSON.parse(process.env.npm_config_argv).original);
if (argv['not-in-browser']) {
argv.b = argv['in-browser'] = false;
}
if (argv['not-in-node']) {
argv.n = argv['in-node'] = false;
}
var commands = [];
if (argv['check-upstream']) {
commands.push(['node', 'scripts/generate/yaml_tests/index.js']);
}
if (argv.unit) {
if (argv['in-node']) {
commands.push(['mocha', 'test/unit/test_*.js', '--require=should']);
}
if (argv['in-browser']) {
commands.push(['testling', '.']);
}
}
if (argv.integration) {
if (argv['in-node']) {
commands.push(['mocha', 'test/integration/yaml_suite/index.js', '-b', '--require=should']);
}
if (argv['in-browser']) {
commands.push(['node', 'scripts/run_browser_integration_suite/index.js']);
}
}
if (commands.length) {
async.forEachSeries(commands, function (args, done) {
var command = args.shift();
console.log(chalk.gray('\n\n' + '# ' + command + ' ' + args.join(' ')));
var proc = cp.spawn(command, args, {
stdio: 'inherit'
});
proc.on('error', function (err) {
proc.removeAllListeners();
done(err);
});
proc.on('exit', function (status) {
proc.removeAllListeners();
done(status ? new Error(command + ' exited with status ' + status) : void 0);
});
}, function (err) {
if (err) {
console.error(err.message);
process.exit(1);
} else {
process.exit(0);
}
});
} else {
console.log('Arguments resulted in no tests to run.');
console.log('Try combining test types with environments');
}

View File

@ -1,28 +0,0 @@
var http = require('http');
var server = http.createServer(function (req, resp) {
var closed, count = 0;
resp.on('close', function () {
closed = true;
console.log('response was closed');
});
process.removeAllListeners();
var interval = setInterval(function () {
if (count > 99 || resp.closed || closed) {
clearInterval(interval);
console.log('done writing', resp.socket.bytesWritten, 'bytes');
resp.end();
} else {
process.stdout.write('->');
resp.write('line of data, more to come... slowly!');
count++;
}
}, 100);
});
server.listen(7500, function () {
console.log('server listening at', server.address());
});