yaroslav
/
SequenceDiagram
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add line numbers to errors [#15]

This commit is contained in:
David Evans 2017-11-08 19:57:41 +00:00
parent 2cb34c273c
commit e6064b72de
7 changed files with 342 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/sequence/CodeMirrorMode.js
View File

@ -153,12 +153,15 @@ define(['core/ArrayUtilities'], (array) => {
}},
'define': {type: 'keyword', suggest: true, then: {
'': aliasListToEnd,
'as': CM_ERROR,
}},
'begin': {type: 'keyword', suggest: true, then: {
'': aliasListToEnd,
'as': CM_ERROR,
}},
'end': {type: 'keyword', suggest: true, then: {
'': aliasListToEnd,
'as': CM_ERROR,
'\n': end,
}},
'if': {type: 'keyword', suggest: true, then: {

22
scripts/sequence/Generator.js
View File

@ -323,7 +323,7 @@ define(['core/ArrayUtilities'], (array) => {
};
}
beginNested(mode, label, name) {
beginNested(mode, label, name, ln) {
const leftAgent = makeAgent(name + '[', {anchorRight: true});
const rightAgent = makeAgent(name + ']');
const agents = [leftAgent, rightAgent];
@ -332,6 +332,7 @@ define(['core/ArrayUtilities'], (array) => {
mode,
label,
stages,
ln,
};
this.currentNest = {
agents,
@ -457,13 +458,13 @@ define(['core/ArrayUtilities'], (array) => {
]);
}
handleBlockBegin({mode, label}) {
handleBlockBegin({ln, mode, label}) {
const name = '__BLOCK' + this.blockCount;
this.beginNested(mode, label, name);
this.beginNested(mode, label, name, ln);
++ this.blockCount;
}
handleBlockSplit({mode, label}) {
handleBlockSplit({ln, mode, label}) {
const containerMode = this.currentNest.stage.sections[0].mode;
if(containerMode !== 'if') {
throw new Error(
@ -476,6 +477,7 @@ define(['core/ArrayUtilities'], (array) => {
mode,
label,
stages: [],
ln,
};
this.currentNest.stage.sections.push(this.currentSection);
}
@ -501,7 +503,13 @@ define(['core/ArrayUtilities'], (array) => {
}
handleStage(stage) {
try {
this.stageHandlers[stage.type](stage);
} catch(e) {
if(typeof e === 'object' && e.message) {
throw new Error(e.message + ' at line ' + (stage.ln + 1));
}
}
}
generate({stages, meta = {}}) {
@ -511,14 +519,14 @@ define(['core/ArrayUtilities'], (array) => {
this.agents.length = 0;
this.blockCount = 0;
this.nesting.length = 0;
const globals = this.beginNested('global', '', '');
const globals = this.beginNested('global', '', '', 0);
stages.forEach(this.handleStage);
if(this.nesting.length !== 1) {
throw new Error(
'Invalid block nesting (' +
(this.nesting.length - 1) + ' unclosed)'
'Unterminated section at line ' +
(this.currentSection.ln + 1)
);
}

28
scripts/sequence/Generator_spec.js
View File

@ -14,12 +14,12 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => {
}
const PARSED = {
blockBegin: (mode, label) => {
return {type: 'block begin', mode, label};
blockBegin: (mode, label, {ln = 0} = {}) => {
return {type: 'block begin', mode, label, ln};
},
blockSplit: (mode, label) => {
return {type: 'block split', mode, label};
blockSplit: (mode, label, {ln = 0} = {}) => {
return {type: 'block split', mode, label, ln};
},
blockEnd: () => {
@ -662,20 +662,20 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => {
it('records all sections within blocks', () => {
const sequence = generator.generate({stages: [
PARSED.blockBegin('if', 'abc'),
PARSED.blockBegin('if', 'abc', {ln: 10}),
PARSED.connect(['A', 'B']),
PARSED.blockSplit('else', 'xyz'),
PARSED.blockSplit('else', 'xyz', {ln: 20}),
PARSED.connect(['A', 'C']),
PARSED.blockEnd(),
]});
const block0 = sequence.stages[0];
expect(block0.sections).toEqual([
{mode: 'if', label: 'abc', stages: [
{mode: 'if', label: 'abc', ln: 10, stages: [
GENERATED.beginAgents(['A', 'B']),
GENERATED.connect(['A', 'B']),
]},
{mode: 'else', label: 'xyz', stages: [
{mode: 'else', label: 'xyz', ln: 20, stages: [
GENERATED.beginAgents(['C']),
GENERATED.connect(['A', 'C']),
]},
@ -756,11 +756,11 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => {
const block0 = sequence.stages[0];
expect(block0.sections).toEqual([
{mode: 'if', label: 'abc', stages: [
{mode: 'if', label: 'abc', ln: 0, stages: [
jasmine.anything(),
jasmine.anything(),
]},
{mode: 'else', label: 'xyz', stages: []},
{mode: 'else', label: 'xyz', ln: 0, stages: []},
]);
});
@ -774,8 +774,8 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => {
const block0 = sequence.stages[0];
expect(block0.sections).toEqual([
{mode: 'if', label: 'abc', stages: []},
{mode: 'else', label: 'xyz', stages: [
{mode: 'if', label: 'abc', ln: 0, stages: []},
{mode: 'else', label: 'xyz', ln: 0, stages: [
jasmine.anything(),
jasmine.anything(),
]},
@ -832,11 +832,11 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => {
const block0 = sequence.stages[0];
expect(block0.sections).toEqual([
{mode: 'if', label: 'abc', stages: [
{mode: 'if', label: 'abc', ln: 0, stages: [
jasmine.anything(),
jasmine.anything(),
]},
{mode: 'else', label: 'xyz', stages: []},
{mode: 'else', label: 'xyz', ln: 0, stages: []},
]);
});

64
scripts/sequence/Parser.js
View File

@ -71,6 +71,28 @@ define([
'end': {type: 'agent end', mode: 'cross'},
};
function makeError(message, token = null) {
let suffix = '';
if(token) {
suffix = (
' at line ' + (token.b.ln + 1) +
', character ' + token.b.ch
);
}
return new Error(message + suffix);
}
function errToken(line, pos) {
if(pos < line.length) {
return line[pos];
}
const last = array.last(line);
if(!last) {
return null;
}
return {b: last.e};
}
function joinLabel(line, begin = 0, end = null) {
if(end === null) {
end = line.length;
@ -93,16 +115,20 @@ define([
}
function skipOver(line, start, skip, error = null) {
const pass = skip.every((expected, i) => (
tokenKeyword(line[start + i]) === expected
));
if(!pass) {
for(let i = 0; i < skip.length; ++ i) {
const expected = skip[i];
const token = line[start + i];
if(tokenKeyword(token) !== expected) {
if(error) {
throw new Error(error + ': ' + joinLabel(line));
throw makeError(
error + '; expected "' + expected + '"',
token
);
} else {
return start;
}
}
}
return start + skip.length;
}
@ -124,7 +150,7 @@ define([
aliasSep = end;
}
if(start >= aliasSep) {
throw new Error('Missing agent name');
throw makeError('Missing agent name', errToken(line, start));
}
return {
name: joinLabel(line, start, aliasSep),
@ -139,11 +165,12 @@ define([
const flags = [];
let p = start;
for(; p < end; ++ p) {
const rawFlag = tokenKeyword(line[p]);
const token = line[p];
const rawFlag = tokenKeyword(token);
const flag = flagTypes[rawFlag];
if(flag) {
if(flags.includes(flag)) {
throw new Error('Duplicate agent flag: ' + rawFlag);
throw makeError('Duplicate agent flag: ' + rawFlag, token);
}
flags.push(flag);
} else {
@ -204,7 +231,7 @@ define([
const type = tokenKeyword(line[1]);
if(TERMINATOR_TYPES.indexOf(type) === -1) {
throw new Error('Unknown termination: ' + joinLabel(line));
throw makeError('Unknown termination "' + type + '"', line[1]);
}
meta.terminators = type;
return true;
@ -274,14 +301,11 @@ define([
let skip = 2;
skip = skipOver(line, skip, type.skip);
const agents = readAgentList(line, skip, labelSep);
if(
agents.length < type.min ||
(type.max !== null && agents.length > type.max)
) {
throw new Error(
'Invalid ' + mode.mode +
': ' + joinLabel(line)
);
if(agents.length < type.min) {
throw makeError('Too few agents for ' + mode.mode, line[0]);
}
if(type.max !== null && agents.length > type.max) {
throw makeError('Too many agents for ' + mode.mode, line[0]);
}
return {
type: type.type,
@ -343,9 +367,13 @@ define([
}
}
if(!stage) {
throw new Error('Unrecognised command: ' + joinLabel(line));
throw makeError(
'Unrecognised command: ' + joinLabel(line),
line[0]
);
}
if(typeof stage === 'object') {
stage.ln = line[0].b.ln;
stages.push(stage);
}
}

105
scripts/sequence/Parser_spec.js
View File

@ -4,7 +4,43 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
const parser = new Parser();
const PARSED = {
blockBegin: ({
ln = jasmine.anything(),
mode = jasmine.anything(),
label = jasmine.anything(),
} = {}) => {
return {
type: 'block begin',
ln,
mode,
label,
};
},
blockSplit: ({
ln = jasmine.anything(),
mode = jasmine.anything(),
label = jasmine.anything(),
} = {}) => {
return {
type: 'block split',
ln,
mode,
label,
};
},
blockEnd: ({
ln = jasmine.anything(),
} = {}) => {
return {
type: 'block end',
ln,
};
},
connect: (agentNames, {
ln = jasmine.anything(),
line = jasmine.anything(),
left = jasmine.anything(),
right = jasmine.anything(),
@ -12,6 +48,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
} = {}) => {
return {
type: 'connect',
ln,
agents: agentNames.map((name) => ({
name,
alias: '',
@ -77,7 +114,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
it('propagates aliases', () => {
const parsed = parser.parse('define Foo Bar as A B');
expect(parsed.stages).toEqual([
{type: 'agent define', agents: [
{type: 'agent define', ln: jasmine.anything(), agents: [
{name: 'Foo Bar', alias: 'A B', flags: []},
]},
]);
@ -102,6 +139,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
expect(parsed.stages).toEqual([
{
type: 'connect',
ln: jasmine.anything(),
agents: [
{name: 'A', alias: '', flags: ['start']},
{name: 'B', alias: '', flags: [
@ -117,12 +155,18 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
});
it('rejects duplicate flags', () => {
expect(() => parser.parse('A -> +*+B')).toThrow();
expect(() => parser.parse('A -> **B')).toThrow();
expect(() => parser.parse('A -> +*+B')).toThrow(new Error(
'Duplicate agent flag: + at line 1, character 7'
));
expect(() => parser.parse('A -> **B')).toThrow(new Error(
'Duplicate agent flag: * at line 1, character 6'
));
});
it('rejects missing agent names', () => {
expect(() => parser.parse('A -> +')).toThrow();
expect(() => parser.parse('A -> +')).toThrow(new Error(
'Missing agent name at line 1, character 6'
));
});
it('converts multiple entries', () => {
@ -141,6 +185,14 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
]);
});
it('stores line numbers', () => {
const parsed = parser.parse('A -> B\nB -> A');
expect(parsed.stages).toEqual([
PARSED.connect(['A', 'B'], {ln: 0}),
PARSED.connect(['B', 'A'], {ln: 1}),
]);
});
it('recognises all types of connection', () => {
const parsed = parser.parse(
'A -> B\n' +
@ -215,6 +267,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
const parsed = parser.parse('note over A: hello there');
expect(parsed.stages).toEqual([{
type: 'note over',
ln: jasmine.anything(),
agents: [{name: 'A', alias: '', flags: []}],
mode: 'note',
label: 'hello there',
@ -232,30 +285,35 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
expect(parsed.stages).toEqual([
{
type: 'note left',
ln: jasmine.anything(),
agents: [{name: 'A', alias: '', flags: []}],
mode: 'note',
label: 'hello there',
},
{
type: 'note left',
ln: jasmine.anything(),
agents: [{name: 'A', alias: '', flags: []}],
mode: 'note',
label: 'hello there',
},
{
type: 'note right',
ln: jasmine.anything(),
agents: [{name: 'A', alias: '', flags: []}],
mode: 'note',
label: 'hello there',
},
{
type: 'note right',
ln: jasmine.anything(),
agents: [{name: 'A', alias: '', flags: []}],
mode: 'note',
label: 'hello there',
},
{
type: 'note between',
ln: jasmine.anything(),
agents: [
{name: 'A', alias: '', flags: []},
{name: 'B', alias: '', flags: []},
@ -270,6 +328,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
const parsed = parser.parse('note over A B, C D: hi');
expect(parsed.stages).toEqual([{
type: 'note over',
ln: jasmine.anything(),
agents: [
{name: 'A B', alias: '', flags: []},
{name: 'C D', alias: '', flags: []},
@ -280,13 +339,16 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
});
it('rejects note between for a single agent', () => {
expect(() => parser.parse('note between A: hi')).toThrow();
expect(() => parser.parse('note between A: hi')).toThrow(new Error(
'Too few agents for note at line 1, character 0'
));
});
it('converts state', () => {
const parsed = parser.parse('state over A: doing stuff');
expect(parsed.stages).toEqual([{
type: 'note over',
ln: jasmine.anything(),
agents: [{name: 'A', alias: '', flags: []}],
mode: 'state',
label: 'doing stuff',
@ -294,13 +356,16 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
});
it('rejects multiple agents for state', () => {
expect(() => parser.parse('state over A, B: hi')).toThrow();
expect(() => parser.parse('state over A, B: hi')).toThrow(new Error(
'Too many agents for state at line 1, character 0'
));
});
it('converts text blocks', () => {
const parsed = parser.parse('text right of A: doing stuff');
expect(parsed.stages).toEqual([{
type: 'note right',
ln: jasmine.anything(),
agents: [{name: 'A', alias: '', flags: []}],
mode: 'text',
label: 'doing stuff',
@ -316,6 +381,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
expect(parsed.stages).toEqual([
{
type: 'agent define',
ln: jasmine.anything(),
agents: [
{name: 'A', alias: '', flags: []},
{name: 'B', alias: '', flags: []},
@ -323,6 +389,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
},
{
type: 'agent begin',
ln: jasmine.anything(),
agents: [
{name: 'A', alias: '', flags: []},
{name: 'B', alias: '', flags: []},
@ -331,6 +398,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
},
{
type: 'agent end',
ln: jasmine.anything(),
agents: [
{name: 'A', alias: '', flags: []},
{name: 'B', alias: '', flags: []},
@ -344,6 +412,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
const parsed = parser.parse('abc:');
expect(parsed.stages).toEqual([{
type: 'mark',
ln: jasmine.anything(),
name: 'abc',
}]);
});
@ -352,6 +421,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
const parsed = parser.parse('simultaneously:');
expect(parsed.stages).toEqual([{
type: 'async',
ln: jasmine.anything(),
target: '',
}]);
});
@ -360,6 +430,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
const parsed = parser.parse('simultaneously with abc:');
expect(parsed.stages).toEqual([{
type: 'async',
ln: jasmine.anything(),
target: 'abc',
}]);
});
@ -376,21 +447,21 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
'end\n'
);
expect(parsed.stages).toEqual([
{type: 'block begin', mode: 'if', label: 'something happens'},
PARSED.blockBegin({mode: 'if', label: 'something happens'}),
PARSED.connect(['A', 'B']),
{type: 'block split', mode: 'else', label: 'something else'},
PARSED.blockSplit({mode: 'else', label: 'something else'}),
PARSED.connect(['A', 'C']),
PARSED.connect(['C', 'B']),
{type: 'block split', mode: 'else', label: ''},
PARSED.blockSplit({mode: 'else', label: ''}),
PARSED.connect(['A', 'D']),
{type: 'block end'},
PARSED.blockEnd(),
]);
});
it('converts loop blocks', () => {
const parsed = parser.parse('repeat until something');
expect(parsed.stages).toEqual([
{type: 'block begin', mode: 'repeat', label: 'until something'},
PARSED.blockBegin({mode: 'repeat', label: 'until something'}),
]);
});
@ -399,7 +470,9 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
});
it('rejects invalid inputs', () => {
expect(() => parser.parse('huh')).toThrow();
expect(() => parser.parse('huh')).toThrow(new Error(
'Unrecognised command: huh at line 1, character 0'
));
});
it('rejects partial links', () => {
@ -409,7 +482,9 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
});
it('rejects invalid terminators', () => {
expect(() => parser.parse('terminators foo')).toThrow();
expect(() => parser.parse('terminators foo')).toThrow(new Error(
'Unknown termination "foo" at line 1, character 12'
));
});
it('rejects malformed notes', () => {
@ -417,7 +492,9 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => {
});
it('rejects malformed block commands', () => {
expect(() => parser.parse('else nope foo')).toThrow();
expect(() => parser.parse('else nope foo')).toThrow(new Error(
'Invalid block command; expected "if" at line 1, character 5'
));
});
it('rejects invalid notes', () => {

101
scripts/sequence/Tokeniser.js
View File

@ -104,30 +104,93 @@ define(['./CodeMirrorMode'], (CMMode) => {
}
}
function copyPos(pos) {
return {i: pos.i, ln: pos.ln, ch: pos.ch};
}
function advancePos(pos, src, steps) {
for(let i = 0; i < steps; ++ i) {
++ pos.ch;
if(src[pos.i + i] === '\n') {
++ pos.ln;
pos.ch = 0;
}
}
pos.i += steps;
}
class TokenState {
constructor(src) {
this.src = src;
this.block = null;
this.token = null;
this.pos = {i: 0, ln: 0, ch: 0};
this.reset();
}
isOver() {
return this.pos.i > this.src.length;
}
reset() {
this.token = {s: '', v: '', q: false, b: null, e: null};
this.block = null;
}
beginToken(advance) {
this.block = advance.newBlock;
Object.assign(this.token, this.block.baseToken);
this.token.b = copyPos(this.pos);
}
endToken() {
let token = null;
if(!this.block.omit) {
this.token.e = copyPos(this.pos);
token = this.token;
}
this.reset();
return token;
}
advance() {
const advance = tokAdvance(this.src, this.pos.i, this.block);
if(advance.newBlock) {
this.beginToken(advance);
}
this.token.s += advance.appendSpace;
this.token.v += advance.appendValue;
advancePos(this.pos, this.src, advance.skip);
if(advance.end) {
return this.endToken();
} else {
return null;
}
}
}
function posStr(pos) {
return 'line ' + (pos.ln + 1) + ', character ' + pos.ch;
}
return class Tokeniser {
tokenise(src) {
const tokens = [];
let block = null;
let current = {s: '', v: '', q: false};
for(let i = 0; i <= src.length;) {
const advance = tokAdvance(src, i, block);
if(advance.newBlock) {
block = advance.newBlock;
Object.assign(current, block.baseToken);
}
current.s += advance.appendSpace;
current.v += advance.appendValue;
i += advance.skip;
if(advance.end) {
if(!block.omit) {
tokens.push(current);
}
current = {s: '', v: '', q: false};
block = null;
const state = new TokenState(src);
while(!state.isOver()) {
const token = state.advance();
if(token) {
tokens.push(token);
}
}
if(block) {
throw new Error('Unterminated block');
if(state.block) {
throw new Error(
'Unterminated literal (began at ' +
posStr(state.token.b) + ')'
);
}
return tokens;
}

120
scripts/sequence/Tokeniser_spec.js
View File

@ -4,14 +4,24 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const tokeniser = new Tokeniser();
describe('.tokenise', () => {
function token({
s = jasmine.anything(), // spacing
v = jasmine.anything(), // value
q = jasmine.anything(), // isQuote?
b = jasmine.anything(), // begin position
e = jasmine.anything(), // end position
}) {
return {s, v, q, b, e};
}
it('converts the source into atomic tokens', () => {
const input = 'foo bar -> baz';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: ' ', v: 'bar', q: false},
{s: ' ', v: '->', q: false},
{s: ' ', v: 'baz', q: false},
token({s: '', v: 'foo'}),
token({s: ' ', v: 'bar'}),
token({s: ' ', v: '->'}),
token({s: ' ', v: 'baz'}),
]);
});
@ -19,10 +29,21 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = 'foo bar->baz';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: ' ', v: 'bar', q: false},
{s: '', v: '->', q: false},
{s: '', v: 'baz', q: false},
token({s: '', v: 'foo'}),
token({s: ' ', v: 'bar'}),
token({s: '', v: '->'}),
token({s: '', v: 'baz'}),
]);
});
it('stores character numbers', () => {
const input = 'foo bar -> baz';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
token({b: {i: 0, ln: 0, ch: 0}, e: {i: 3, ln: 0, ch: 3}}),
token({b: {i: 4, ln: 0, ch: 4}, e: {i: 7, ln: 0, ch: 7}}),
token({b: {i: 8, ln: 0, ch: 8}, e: {i: 10, ln: 0, ch: 10}}),
token({b: {i: 11, ln: 0, ch: 11}, e: {i: 14, ln: 0, ch: 14}}),
]);
});
@ -30,10 +51,21 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = 'foo bar\nbaz';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: ' ', v: 'bar', q: false},
{s: '', v: '\n', q: false},
{s: '', v: 'baz', q: false},
token({s: '', v: 'foo'}),
token({s: ' ', v: 'bar'}),
token({s: '', v: '\n'}),
token({s: '', v: 'baz'}),
]);
});
it('stores line numbers', () => {
const input = 'foo bar\nbaz';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
token({b: {i: 0, ln: 0, ch: 0}, e: {i: 3, ln: 0, ch: 3}}),
token({b: {i: 4, ln: 0, ch: 4}, e: {i: 7, ln: 0, ch: 7}}),
token({b: {i: 7, ln: 0, ch: 7}, e: {i: 8, ln: 1, ch: 0}}),
token({b: {i: 8, ln: 1, ch: 0}, e: {i: 11, ln: 1, ch: 3}}),
]);
});
@ -41,9 +73,9 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = 'foo "\n" baz';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: ' ', v: '\n', q: true},
{s: ' ', v: 'baz', q: false},
token({s: '', v: 'foo', q: false}),
token({s: ' ', v: '\n', q: true}),
token({s: ' ', v: 'baz', q: false}),
]);
});
@ -51,10 +83,10 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = ' foo \t bar\t\n baz';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: ' ', v: 'foo', q: false},
{s: ' \t ', v: 'bar', q: false},
{s: '\t', v: '\n', q: false},
{s: ' ', v: 'baz', q: false},
token({s: ' ', v: 'foo'}),
token({s: ' \t ', v: 'bar'}),
token({s: '\t', v: '\n'}),
token({s: ' ', v: 'baz'}),
]);
});
@ -62,9 +94,17 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = 'foo "zig zag" \'abc def\'';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: ' ', v: 'zig zag', q: true},
{s: ' ', v: 'abc def', q: true},
token({s: '', v: 'foo', q: false}),
token({s: ' ', v: 'zig zag', q: true}),
token({s: ' ', v: 'abc def', q: true}),
]);
});
it('stores character positions around quoted strings', () => {
const input = '"foo bar"';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
token({b: {i: 0, ln: 0, ch: 0}, e: {i: 9, ln: 0, ch: 9}}),
]);
});
@ -72,9 +112,9 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = 'foo # bar baz\nzig';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: '', v: '\n', q: false},
{s: '', v: 'zig', q: false},
token({s: '', v: 'foo'}),
token({s: '', v: '\n'}),
token({s: '', v: 'zig'}),
]);
});
@ -82,9 +122,9 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = 'foo # bar "\'baz\nzig';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: '', v: '\n', q: false},
{s: '', v: 'zig', q: false},
token({s: '', v: 'foo'}),
token({s: '', v: '\n'}),
token({s: '', v: 'zig'}),
]);
});
@ -92,8 +132,8 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = 'foo "zig\\" zag\\n"';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: ' ', v: 'zig" zag\n', q: true},
token({s: '', v: 'foo', q: false}),
token({s: ' ', v: 'zig" zag\n', q: true}),
]);
});
@ -101,13 +141,27 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => {
const input = 'foo " zig\n zag "';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
{s: '', v: 'foo', q: false},
{s: ' ', v: ' zig\n zag ', q: true},
token({s: '', v: 'foo', q: false}),
token({s: ' ', v: ' zig\n zag ', q: true}),
]);
});
it('calculates line numbers consistently within quotes', () => {
const input = 'foo\nbar "zig\nzag\na" b';
const tokens = tokeniser.tokenise(input);
expect(tokens).toEqual([
token({b: {i: 0, ln: 0, ch: 0}, e: {i: 3, ln: 0, ch: 3}}),
token({b: {i: 3, ln: 0, ch: 3}, e: {i: 4, ln: 1, ch: 0}}),
token({b: {i: 4, ln: 1, ch: 0}, e: {i: 7, ln: 1, ch: 3}}),
token({b: {i: 8, ln: 1, ch: 4}, e: {i: 19, ln: 3, ch: 2}}),
token({b: {i: 20, ln: 3, ch: 3}, e: {i: 21, ln: 3, ch: 4}}),
]);
});
it('rejects unterminated quoted values', () => {
expect(() => tokeniser.tokenise('"nope')).toThrow();
expect(() => tokeniser.tokenise('"nope')).toThrow(new Error(
'Unterminated literal (began at line 1, character 0)'
));
});
});