I'm storing response strings in a database in EJS form and filling out the data in Node. What I want to do is be able to use any property I want, no matter what model it comes from, then in Node, async/await those models once I have the template, based around what properties are required.
So if I have a template like:
"Hello <%=user.firstName%>."
I want to be able to look at that template and extract something like:
ejsProperties = ["user", "user.firstName"]
Or something like that.
Now to use cookies with Express, we will require the cookie-parser. cookie-parser is a middleware which parses cookies attached to the client request object. To use it, we will require it in our index. js file; this can be used the same way as we use other middleware.
If you just want to pull out simple things like user.firstName
then running a RegExp over the EJS file is probably as good a way as any. Chances are you'd be looking for a specific and known set of objects and properties so you could target them specifically rather than trying to extract all possible objects/properties.
In the more general case things get difficult very quickly. Something like this is very tricky to handle:
<% var u = user; %><%= u.firstName %>
It's a silly example but it's just the tip of that particular iceberg. Whereas user
is being read from the locals
and is an object of interest, u
could be just about anything and we can't easily draw lines connecting firstName
and user
via u
. Similarly something like a forEach
on an array or a for/in
on an object will quickly make it impossible to link properties to the appropriate locals
entry.
However, what we can do is identify the entries in locals
, or at least something very close to that.
Using the example of <%= user.firstName %>
the identifier user
could refer to one of 3 things. Firstly, it could be an entry in the locals
. Secondly, it could be a property of the global object. Thirdly, it could be a variable created within the scope of the template (like u
in the earlier example).
We can't really tell the difference between the first two cases but chances are you can separate out the globals pretty easily. Things like console
and Math
can be identified and discarded.
The third case is the tricky one, telling the difference between an entry in the locals
and a variable in the template, like in this example:
<% users.forEach(function(user) { %>
<%= user.firstName %>
<% }); %>
In this case users
is coming directly from the locals
but user
is not. For us to work that out requires variable scope analysis similar to that found in an IDE.
So here's what I tried:
locals
object. EJS uses with (locals) {...}
internally so there really is no way to know which one it is.I've imaginatively called the result ejsprima
.
I haven't attempted to support all the options that EJS supports, so if you're using custom delimiters or strict mode it won't work. (If you're using strict mode you have to explicitly write locals.user.firstName
in your template anyway, which is crying out to be done via a RegExp instead). It won't attempt to follow any include
calls.
I'd be very surprised if there aren't bugs lurking somewhere, even with some piece of basic JS syntax, but I've tested all of the nasty cases I could think of. Test cases are included.
The EJS used in the main demo can be found at the top of the HTML. I've included a gratuitous example of a 'global write' just to demonstrate what they look like but I'd imagine that they aren't something you'd normally want. The interesting bit is the reads
section.
I developed this against esprima 4 but the best CDN version I could find is 2.7.3. The tests all still pass so it doesn't seem to matter too much.
The only code I included in the JS section of the snippet is for 'ejsprima' itself. To run that in Node you should just need to copy it across and tweak the top and bottom to correct the exports and requires stuff.
// Begin 'ejsprima'
(function(exports) {
//var esprima = require('esprima');
// Simple EJS compiler that throws away the HTML sections and just retains the JavaScript code
exports.compile = function(tpl) {
// Extract the tags
var tags = tpl.match(/(<%(?!%)[\s\S]*?[^%]%>)/g);
return tags.map(function(tag) {
var parse = tag.match(/^(<%[=\-_#]?)([\s\S]*?)([-_]?%>)$/);
switch (parse[1]) {
case '<%=':
case '<%-':
return ';(' + parse[2] + ');';
case '<%#':
return '';
case '<%':
case '<%_':
return parse[2];
}
throw new Error('Assertion failure');
}).join('\n');
};
// Pull out the identifiers for all 'global' reads and writes
exports.extractGlobals = function(tpl) {
var ast = tpl;
if (typeof tpl === 'string') {
// Note: This should be parseScript in esprima 4
ast = esprima.parse(tpl);
}
// Uncomment this line to dump out the AST
//console.log(JSON.stringify(ast, null, 2));
var refs = this.processAst(ast);
var reads = {};
var writes = {};
refs.forEach(function(ref) {
ref.globalReads.forEach(function(key) {
reads[key] = true;
});
});
refs.forEach(function(ref) {
ref.globalWrites.forEach(function(key) {
writes[key] = true;
})
});
return {
reads: Object.keys(reads),
writes: Object.keys(writes)
};
};
exports.processAst = function(obj) {
var baseScope = {
lets: Object.create(null),
reads: Object.create(null),
writes: Object.create(null),
vars: Object.assign(Object.create(null), {
// These are all local to the rendering function
arguments: true,
escapeFn: true,
include: true,
rethrow: true
})
};
var scopes = [baseScope];
processNode(obj, baseScope);
scopes.forEach(function(scope) {
scope.globalReads = Object.keys(scope.reads).filter(function(key) {
return !scope.vars[key] && !scope.lets[key];
});
scope.globalWrites = Object.keys(scope.writes).filter(function(key) {
return !scope.vars[key] && !scope.lets[key];
});
// Flatten out the prototype chain - none of this is actually used by extractGlobals so we could just skip it
var allVars = Object.keys(scope.vars).concat(Object.keys(scope.lets)),
vars = {},
lets = {};
// An identifier can either be a var or a let not both... need to ensure inheritance sees the right one by
// setting the alternative to false, blocking any inherited value
for (var key in scope.lets) {
if (hasOwn(scope.lets)) {
scope.vars[key] = false;
}
}
for (key in scope.vars) {
if (hasOwn(scope.vars)) {
scope.lets[key] = false;
}
}
for (key in scope.lets) {
if (scope.lets[key]) {
lets[key] = true;
}
}
for (key in scope.vars) {
if (scope.vars[key]) {
vars[key] = true;
}
}
scope.lets = Object.keys(lets);
scope.vars = Object.keys(vars);
scope.reads = Object.keys(scope.reads);
function hasOwn(obj) {
return obj[key] && (Object.prototype.hasOwnProperty.call(obj, key));
}
});
return scopes;
function processNode(obj, scope) {
if (!obj) {
return;
}
if (Array.isArray(obj)) {
obj.forEach(function(o) {
processNode(o, scope);
});
return;
}
switch(obj.type) {
case 'Identifier':
scope.reads[obj.name] = true;
return;
case 'VariableDeclaration':
obj.declarations.forEach(function(declaration) {
// Separate scopes for var and let/const
processLValue(declaration.id, scope, obj.kind === 'var' ? scope.vars : scope.lets);
processNode(declaration.init, scope);
});
return;
case 'AssignmentExpression':
processLValue(obj.left, scope, scope.writes);
if (obj.operator !== '=') {
processLValue(obj.left, scope, scope.reads);
}
processNode(obj.right, scope);
return;
case 'UpdateExpression':
processLValue(obj.argument, scope, scope.reads);
processLValue(obj.argument, scope, scope.writes);
return;
case 'FunctionDeclaration':
case 'FunctionExpression':
case 'ArrowFunctionExpression':
var newScope = {
lets: Object.create(scope.lets),
reads: Object.create(null),
vars: Object.create(scope.vars),
writes: Object.create(null)
};
scopes.push(newScope);
obj.params.forEach(function(param) {
processLValue(param, newScope, newScope.vars);
});
if (obj.id) {
// For a Declaration the name is accessible outside, for an Expression it is only available inside
if (obj.type === 'FunctionDeclaration') {
scope.vars[obj.id.name] = true;
}
else {
newScope.vars[obj.id.name] = true;
}
}
processNode(obj.body, newScope);
return;
case 'BlockStatement':
case 'CatchClause':
case 'ForInStatement':
case 'ForOfStatement':
case 'ForStatement':
// Create a new block scope
scope = {
lets: Object.create(scope.lets),
reads: Object.create(null),
vars: scope.vars,
writes: Object.create(null)
};
scopes.push(scope);
if (obj.type === 'CatchClause') {
processLValue(obj.param, scope, scope.lets);
processNode(obj.body, scope);
return;
}
break; // Don't return
}
Object.keys(obj).forEach(function(key) {
var value = obj[key];
// Labels for break/continue
if (key === 'label') {
return;
}
if (key === 'left') {
if (obj.type === 'ForInStatement' || obj.type === 'ForOfStatement') {
if (obj.left.type !== 'VariableDeclaration') {
processLValue(obj.left, scope, scope.writes);
return;
}
}
}
if (obj.computed === false) {
// MemberExpression, ClassExpression & Property
if (key === 'property' || key === 'key') {
return;
}
}
if (value && typeof value === 'object') {
processNode(value, scope);
}
});
}
// An l-value is something that can appear on the left of an = operator. It could be a simple identifier, as in
// `var a = 7;`, or something more complicated, like a destructuring. There's a big difference between how we handle
// `var a = 7;` and `a = 7;` and the 'target' is used to control which of these two scenarios we are in.
function processLValue(obj, scope, target) {
nextLValueNode(obj);
function nextLValueNode(obj) {
switch (obj.type) {
case 'Identifier':
target[obj.name] = true;
break;
case 'ObjectPattern':
obj.properties.forEach(function(property) {
if (property.computed) {
processNode(property.key, scope);
}
nextLValueNode(property.value);
});
break;
case 'ArrayPattern':
obj.elements.forEach(function(element) {
nextLValueNode(element);
});
break;
case 'RestElement':
nextLValueNode(obj.argument);
break;
case 'AssignmentPattern':
nextLValueNode(obj.left);
processNode(obj.right, scope);
break;
case 'MemberExpression':
processNode(obj, scope);
break;
default: throw new Error('Unknown type: ' + obj.type);
}
}
}
};
})(window.ejsprima = {});
<body>
<script type="text/ejs" id="demo-ejs">
<body>
<h1>Welcome <%= user.name %></h1>
<% if (admin) { %>
<a href="/admin">Admin</a>
<% } %>
<ul>
<% friends.forEach(function(friend, index) { %>
<li class="<%= index === 0 ? "first" : "" %> <%= friend.name === selected ? "selected" : "" %>"><%= friend.name %></li>
<% }); %>
</ul>
<%
console.log(user);
exampleWrite = 'some value';
%>
</body>
</script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/esprima/2.7.3/esprima.min.js"></script>
<script>
function runTests() {
var assertValues = function(tpl, reads, writes) {
var program = ejsprima.compile(tpl);
var values = ejsprima.extractGlobals(program);
reads = reads || [];
writes = writes || [];
reads.sort();
writes.sort();
if (!equal(reads, values.reads)) {
console.log('Mismatched reads', reads, values.reads, tpl);
}
if (!equal(writes, values.writes)) {
console.log('Mismatched writes', writes, values.writes, tpl);
}
function equal(arr1, arr2) {
return JSON.stringify(arr1.slice().sort()) === JSON.stringify(arr2.slice().sort());
}
};
assertValues('<% console.log("hello") %>', ['console']);
assertValues('<% a = 7; %>', [], ['a']);
assertValues('<% var a = 7; %>');
assertValues('<% let a = 7; %>');
assertValues('<% const a = 7; %>');
assertValues('<% a = 7; var a; %>');
assertValues('<% var a = 7, b = a + 1, c = d; %>', ['d']);
assertValues('<% try{}catch(a){a.log()} %>');
assertValues('<% try{}catch(a){a = 9;} %>');
assertValues('<% try{}catch(a){b.log()} %>', ['b']);
assertValues('<% try{}catch(a){}a; %>', ['a']);
assertValues('<% try{}catch(a){let b;}b; %>', ['b']);
assertValues('<% try{}finally{let a;}a; %>', ['a']);
assertValues('<% (function(a){a();b();}) %>', ['b']);
assertValues('<% (function(a){a();b = 8;}) %>', [], ['b']);
assertValues('<% (function(a){a();a = 8;}) %>');
assertValues('<% (function name(a){}) %>');
assertValues('<% (function name(a){});name(); %>', ['name']);
assertValues('<% function name(a){} %>');
assertValues('<% function name(a){}name(); %>');
assertValues('<% a.map(b => b + c); %>', ['a', 'c']);
assertValues('<% a.map(b => b + c); b += 6; %>', ['a', 'b', 'c'], ['b']);
assertValues('<% var {a} = {b: c}; %>', ['c']);
assertValues('<% var {a} = {b: c}; a(); %>', ['c']);
assertValues('<% var {[d]: a} = {b: c}; a(); %>', ['c', 'd']);
assertValues('<% var {[d]: a} = {b: c}; a(); %>', ['c', 'd']);
assertValues('<% var {[d + e]: a} = {b: c}; a(); %>', ['c', 'd', 'e']);
assertValues('<% var {[d + e[f = g]]: a} = {b: c}; a(); %>', ['c', 'd', 'e', 'g'], ['f']);
assertValues('<% ({a} = {b: c}); %>', ['c'], ['a']);
assertValues('<% ({a: d.e} = {b: c}); %>', ['c', 'd']);
assertValues('<% ({[a]: d.e} = {b: c}); %>', ['a', 'c', 'd']);
assertValues('<% var {a = 7} = {}; %>', []);
assertValues('<% var {a = b} = {}; %>', ['b']);
assertValues('<% var {[a]: b = (c + d)} = {}; %>', ['a', 'c', 'd']);
assertValues('<% var [a] = [b]; a(); %>', ['b']);
assertValues('<% var [{a}] = [b]; a(); %>', ['b']);
assertValues('<% [{a}] = [b]; %>', ['b'], ['a']);
assertValues('<% [...a] = [b]; %>', ['b'], ['a']);
assertValues('<% let [...a] = [b]; %>', ['b']);
assertValues('<% var [a = b] = [c]; %>', ['b', 'c']);
assertValues('<% var [a = b] = [c], b; %>', ['c']);
assertValues('<% ++a %>', ['a'], ['a']);
assertValues('<% ++a.b %>', ['a']);
assertValues('<% var a; ++a %>');
assertValues('<% a += 1 %>', ['a'], ['a']);
assertValues('<% var a; a += 1 %>');
assertValues('<% a.b = 7 %>', ['a']);
assertValues('<% a["b"] = 7 %>', ['a']);
assertValues('<% a[b] = 7 %>', ['a', 'b']);
assertValues('<% a[b + c] = 7 %>', ['a', 'b', 'c']);
assertValues('<% var b; a[b + c] = 7 %>', ['a', 'c']);
assertValues('<% a in b; %>', ['a', 'b']);
assertValues('<% "a" in b; %>', ['b']);
assertValues('<% "a" in b.c; %>', ['b']);
assertValues('<% if (a === b) {c();} %>', ['a', 'b', 'c']);
assertValues('<% if (a = b) {c();} else {d = e} %>', ['b', 'c', 'e'], ['a', 'd']);
assertValues('<% a ? b : c %>', ['a', 'b', 'c']);
assertValues('<% var a = b ? c : d %>', ['b', 'c', 'd']);
assertValues('<% for (a in b) {} %>', ['b'], ['a']);
assertValues('<% for (var a in b.c) {} %>', ['b']);
assertValues('<% for (let {a} in b) {} %>', ['b']);
assertValues('<% for ({a} in b) {} %>', ['b'], ['a']);
assertValues('<% for (var {[a + b]: c} in d) {} %>', ['a', 'b', 'd']);
assertValues('<% for ({[a + b]: c} in d) {} %>', ['a', 'b', 'd'], ['c']);
assertValues('<% for (var a in b) {a = a + c;} %>', ['b', 'c']);
assertValues('<% for (const a in b) console.log(a); %>', ['b', 'console']);
assertValues('<% for (let a in b) console.log(a); %>', ['b', 'console']);
assertValues('<% for (let a in b) {let b = 5;} %>', ['b']);
assertValues('<% for (let a in b) {let b = 5;} console.log(a); %>', ['console', 'a', 'b']);
assertValues('<% for (const a in b) {let b = 5;} console.log(a); %>', ['console', 'a', 'b']);
assertValues('<% for (var a in b) {let b = 5;} console.log(a); %>', ['console', 'b']);
assertValues('<% for (a of b) {} %>', ['b'], ['a']);
assertValues('<% for (var a of b.c) {} %>', ['b']);
assertValues('<% for (let {a} of b) {} %>', ['b']);
assertValues('<% for ({a} of b) {} %>', ['b'], ['a']);
assertValues('<% for (var {[a + b]: c} of d) {} %>', ['a', 'b', 'd']);
assertValues('<% for ({[a + b]: c} of d) {} %>', ['a', 'b', 'd'], ['c']);
assertValues('<% for (var a of b) {a = a + c;} %>', ['b', 'c']);
assertValues('<% for (const a of b) console.log(a); %>', ['b', 'console']);
assertValues('<% for (let a of b) console.log(a); %>', ['b', 'console']);
assertValues('<% for (let a of b) {let b = 5;} %>', ['b']);
assertValues('<% for (let a of b) {let b = 5;} console.log(a); %>', ['console', 'a', 'b']);
assertValues('<% for (const a of b) {let b = 5;} console.log(a); %>', ['console', 'a', 'b']);
assertValues('<% for (var a of b) {let b = 5;} console.log(a); %>', ['console', 'b']);
assertValues('<% for (var i = 0 ; i < 10 ; ++i) {} %>');
assertValues('<% for (var i = 0 ; i < len ; ++i) {} %>', ['len']);
assertValues('<% for (var i = 0, len ; i < len ; ++i) {} %>');
assertValues('<% for (i = 0 ; i < len ; ++i) {} %>', ['i', 'len'], ['i']);
assertValues('<% for ( ; i < len ; ++i) {} %>', ['i', 'len'], ['i']);
assertValues('<% var i; for ( ; i < len ; ++i) {} %>', ['len']);
assertValues('<% for (var i = 0 ; i < 10 ; ++i) {i += j;} %>', ['j']);
assertValues('<% for (var i = 0 ; i < 10 ; ++i) {j += i;} %>', ['j'], ['j']);
assertValues('<% for (const i = 0; i < 10 ; ++i) console.log(i); %>', ['console']);
assertValues('<% for (let i = 0 ; i < 10 ; ++i) console.log(i); %>', ['console']);
assertValues('<% for (let i = 0 ; i < len ; ++i) {let len = 5;} %>', ['len']);
assertValues('<% for (let i = 0 ; i < len ; ++i) {let len = 5;} console.log(i); %>', ['console', 'i', 'len']);
assertValues('<% for (var i = 0 ; i < len ; ++i) {let len = 5;} console.log(i); %>', ['console', 'len']);
assertValues('<% while(++i){console.log(i);} %>', ['console', 'i'], ['i']);
assertValues('<% myLabel:while(true){break myLabel;} %>');
assertValues('<% var a = `Hello ${user.name}`; %>', ['user']);
assertValues('<% this; null; true; false; NaN; undefined; %>', ['NaN', 'undefined']);
// Scoping
assertValues([
'<%',
'var a = 7, b;',
'let c = 8;',
'a = b + c - d;',
'{',
'let e = 6;',
'f = g + e + b + c;',
'}',
'%>'
].join('\n'), ['d', 'g'], ['f']);
assertValues([
'<%',
'var a = 7, b;',
'let c = 8;',
'a = b + c - d;',
'{',
'let e = 6;',
'f = g + e + b + c;',
'}',
'e = c;',
'%>'
].join('\n'), ['d', 'g'], ['e', 'f']);
assertValues([
'<%',
'var a = 7, b;',
'let c = 8;',
'a = b + c - d;',
'{',
'var e = 6;',
'f = g + e + b + c;',
'}',
'e = c;',
'%>'
].join('\n'), ['d', 'g'], ['f']);
assertValues([
'<%',
'var a;',
'let b;',
'const c = 0;',
'{',
'var d;',
'let e;',
'const f = 1;',
'}',
'var g = function h(i) {',
'arguments.length;',
'a(); b(); c(); d(); e(); f(); g(); h(); i();',
'};',
'%>'
].join('\n'), ['e', 'f']);
assertValues([
'<%',
'var a;',
'let b;',
'const c = 0;',
'{',
'var d;',
'let e;',
'const f = 1;',
'}',
'var g = function h(i) {};',
'arguments.length;',
'a(); b(); c(); d(); e(); f(); g(); h(); i();',
'%>'
].join('\n'), ['e', 'f', 'h', 'i']);
assertValues([
'<%',
'var a;',
'let b;',
'const c = 0;',
'{',
'var d;',
'let e;',
'const f = 1;',
'arguments.length;',
'a(); b(); c(); d(); e(); f(); g(); h(); i();',
'}',
'var g = function h(i) {};',
'%>'
].join('\n'), ['h', 'i']);
assertValues([
'<%',
'var a;',
'let b;',
'const c = 0;',
'{',
'var d;',
'let e;',
'const f = 1;',
'var g = function h(i) {',
'arguments.length;',
'a(); b(); c(); d(); e(); f(); g(); h(); i();',
'};',
'}',
'%>'
].join('\n'));
assertValues([
'<%',
'var a;',
'let b;',
'const c = 0;',
'var g = function h(i) {',
'{',
'var d;',
'let e;',
'const f = 1;',
'}',
'arguments.length;',
'a(); b(); c(); d(); e(); f(); g(); h(); i();',
'};',
'%>'
].join('\n'), ['e', 'f']);
assertValues([
'<%',
'var a;',
'let b;',
'const c = 0;',
'var g = function h(i) {',
'{',
'var d;',
'let e;',
'const f = 1;',
'arguments.length;',
'a(); b(); c(); d(); e(); f(); g(); h(); i();',
'}',
'};',
'%>'
].join('\n'));
// EJS parsing
assertValues('Hello <%= user.name %>', ['user']);
assertValues('Hello <%- user.name %>', ['user']);
assertValues('Hello <%# user.name %>');
assertValues('Hello <%_ user.name _%>', ['user']);
assertValues('Hello <%_ user.name _%>', ['user']);
assertValues('Hello <%% console.log("<%= user.name %>") %%>', ['user']);
assertValues('Hello <% console.log("<%% user.name %%>") %>', ['console']);
assertValues('<% %><%a%>', ['a']);
assertValues('<% %><%=a%>', ['a']);
assertValues('<% %><%-a_%>', ['a']);
assertValues('<% %><%__%>');
assertValues([
'<body>',
'<h1>Welcome <%= user.name %></h1>',
'<% if (admin) { %>',
'<a href="/admin">Admin</a>',
'<% } %>',
'<ul>',
'<% friends.forEach(function(friend, index) { %>',
'<li class="<%= index === 0 ? "first" : "" %> <%= friend.name === selected ? "selected" : "" %>"><%= friend.name %></li>',
'<% }); %>',
'</ul>',
'</body>'
].join('\n'), ['user', 'admin', 'friends', 'selected']);
assertValues([
'<body>',
'<h1>Welcome <%= user.name %></h1>',
'<% if (admin) { %>',
'<a href="/admin">Admin</a>',
'<% } %>',
'<ul>',
'<% friends.forEach(function(user, index) { %>',
'<li class="<%= index === 0 ? "first" : "" %> <%= user.name === selected ? "selected" : "" %>"><%= user.name %></li>',
'<% }); %>',
'</ul>',
'</body>'
].join('\n'), ['user', 'admin', 'friends', 'selected']);
console.log('Tests complete, if you didn\'t see any other messages then they passed');
}
</script>
<script>
function runDemo() {
var script = document.getElementById('demo-ejs'),
tpl = script.innerText,
js = ejsprima.compile(tpl);
console.log(ejsprima.extractGlobals(js));
}
</script>
<button onclick="runTests()">Run Tests</button>
<button onclick="runDemo()">Run Demo</button>
</body>
So, in summary, I believe this will allow you to accurately identify all the required entries for your locals
. Identifying the properties used within those objects is, in general, not possible. If you don't mind the loss of accuracy then you might as well just use a RegExp.
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With