diff --git a/scripts/sequence/CodeMirrorMode.js b/scripts/sequence/CodeMirrorMode.js index 7ea91a4..4b11bab 100644 --- a/scripts/sequence/CodeMirrorMode.js +++ b/scripts/sequence/CodeMirrorMode.js @@ -153,12 +153,15 @@ define(['core/ArrayUtilities'], (array) => { }}, 'define': {type: 'keyword', suggest: true, then: { '': aliasListToEnd, + 'as': CM_ERROR, }}, 'begin': {type: 'keyword', suggest: true, then: { '': aliasListToEnd, + 'as': CM_ERROR, }}, 'end': {type: 'keyword', suggest: true, then: { '': aliasListToEnd, + 'as': CM_ERROR, '\n': end, }}, 'if': {type: 'keyword', suggest: true, then: { diff --git a/scripts/sequence/Generator.js b/scripts/sequence/Generator.js index fa4c524..1c07dfe 100644 --- a/scripts/sequence/Generator.js +++ b/scripts/sequence/Generator.js @@ -323,7 +323,7 @@ define(['core/ArrayUtilities'], (array) => { }; } - beginNested(mode, label, name) { + beginNested(mode, label, name, ln) { const leftAgent = makeAgent(name + '[', {anchorRight: true}); const rightAgent = makeAgent(name + ']'); const agents = [leftAgent, rightAgent]; @@ -332,6 +332,7 @@ define(['core/ArrayUtilities'], (array) => { mode, label, stages, + ln, }; this.currentNest = { agents, @@ -457,13 +458,13 @@ define(['core/ArrayUtilities'], (array) => { ]); } - handleBlockBegin({mode, label}) { + handleBlockBegin({ln, mode, label}) { const name = '__BLOCK' + this.blockCount; - this.beginNested(mode, label, name); + this.beginNested(mode, label, name, ln); ++ this.blockCount; } - handleBlockSplit({mode, label}) { + handleBlockSplit({ln, mode, label}) { const containerMode = this.currentNest.stage.sections[0].mode; if(containerMode !== 'if') { throw new Error( @@ -476,6 +477,7 @@ define(['core/ArrayUtilities'], (array) => { mode, label, stages: [], + ln, }; this.currentNest.stage.sections.push(this.currentSection); } @@ -501,7 +503,13 @@ define(['core/ArrayUtilities'], (array) => { } handleStage(stage) { - this.stageHandlers[stage.type](stage); + try { + this.stageHandlers[stage.type](stage); + } catch(e) { + if(typeof e === 'object' && e.message) { + throw new Error(e.message + ' at line ' + (stage.ln + 1)); + } + } } generate({stages, meta = {}}) { @@ -511,14 +519,14 @@ define(['core/ArrayUtilities'], (array) => { this.agents.length = 0; this.blockCount = 0; this.nesting.length = 0; - const globals = this.beginNested('global', '', ''); + const globals = this.beginNested('global', '', '', 0); stages.forEach(this.handleStage); if(this.nesting.length !== 1) { throw new Error( - 'Invalid block nesting (' + - (this.nesting.length - 1) + ' unclosed)' + 'Unterminated section at line ' + + (this.currentSection.ln + 1) ); } diff --git a/scripts/sequence/Generator_spec.js b/scripts/sequence/Generator_spec.js index 38275a2..cae6a59 100644 --- a/scripts/sequence/Generator_spec.js +++ b/scripts/sequence/Generator_spec.js @@ -14,12 +14,12 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => { } const PARSED = { - blockBegin: (mode, label) => { - return {type: 'block begin', mode, label}; + blockBegin: (mode, label, {ln = 0} = {}) => { + return {type: 'block begin', mode, label, ln}; }, - blockSplit: (mode, label) => { - return {type: 'block split', mode, label}; + blockSplit: (mode, label, {ln = 0} = {}) => { + return {type: 'block split', mode, label, ln}; }, blockEnd: () => { @@ -662,20 +662,20 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => { it('records all sections within blocks', () => { const sequence = generator.generate({stages: [ - PARSED.blockBegin('if', 'abc'), + PARSED.blockBegin('if', 'abc', {ln: 10}), PARSED.connect(['A', 'B']), - PARSED.blockSplit('else', 'xyz'), + PARSED.blockSplit('else', 'xyz', {ln: 20}), PARSED.connect(['A', 'C']), PARSED.blockEnd(), ]}); const block0 = sequence.stages[0]; expect(block0.sections).toEqual([ - {mode: 'if', label: 'abc', stages: [ + {mode: 'if', label: 'abc', ln: 10, stages: [ GENERATED.beginAgents(['A', 'B']), GENERATED.connect(['A', 'B']), ]}, - {mode: 'else', label: 'xyz', stages: [ + {mode: 'else', label: 'xyz', ln: 20, stages: [ GENERATED.beginAgents(['C']), GENERATED.connect(['A', 'C']), ]}, @@ -756,11 +756,11 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => { const block0 = sequence.stages[0]; expect(block0.sections).toEqual([ - {mode: 'if', label: 'abc', stages: [ + {mode: 'if', label: 'abc', ln: 0, stages: [ jasmine.anything(), jasmine.anything(), ]}, - {mode: 'else', label: 'xyz', stages: []}, + {mode: 'else', label: 'xyz', ln: 0, stages: []}, ]); }); @@ -774,8 +774,8 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => { const block0 = sequence.stages[0]; expect(block0.sections).toEqual([ - {mode: 'if', label: 'abc', stages: []}, - {mode: 'else', label: 'xyz', stages: [ + {mode: 'if', label: 'abc', ln: 0, stages: []}, + {mode: 'else', label: 'xyz', ln: 0, stages: [ jasmine.anything(), jasmine.anything(), ]}, @@ -832,11 +832,11 @@ defineDescribe('Sequence Generator', ['./Generator'], (Generator) => { const block0 = sequence.stages[0]; expect(block0.sections).toEqual([ - {mode: 'if', label: 'abc', stages: [ + {mode: 'if', label: 'abc', ln: 0, stages: [ jasmine.anything(), jasmine.anything(), ]}, - {mode: 'else', label: 'xyz', stages: []}, + {mode: 'else', label: 'xyz', ln: 0, stages: []}, ]); }); diff --git a/scripts/sequence/Parser.js b/scripts/sequence/Parser.js index 2de9327..539aaa3 100644 --- a/scripts/sequence/Parser.js +++ b/scripts/sequence/Parser.js @@ -71,6 +71,28 @@ define([ 'end': {type: 'agent end', mode: 'cross'}, }; + function makeError(message, token = null) { + let suffix = ''; + if(token) { + suffix = ( + ' at line ' + (token.b.ln + 1) + + ', character ' + token.b.ch + ); + } + return new Error(message + suffix); + } + + function errToken(line, pos) { + if(pos < line.length) { + return line[pos]; + } + const last = array.last(line); + if(!last) { + return null; + } + return {b: last.e}; + } + function joinLabel(line, begin = 0, end = null) { if(end === null) { end = line.length; @@ -93,14 +115,18 @@ define([ } function skipOver(line, start, skip, error = null) { - const pass = skip.every((expected, i) => ( - tokenKeyword(line[start + i]) === expected - )); - if(!pass) { - if(error) { - throw new Error(error + ': ' + joinLabel(line)); - } else { - return start; + for(let i = 0; i < skip.length; ++ i) { + const expected = skip[i]; + const token = line[start + i]; + if(tokenKeyword(token) !== expected) { + if(error) { + throw makeError( + error + '; expected "' + expected + '"', + token + ); + } else { + return start; + } } } return start + skip.length; @@ -124,7 +150,7 @@ define([ aliasSep = end; } if(start >= aliasSep) { - throw new Error('Missing agent name'); + throw makeError('Missing agent name', errToken(line, start)); } return { name: joinLabel(line, start, aliasSep), @@ -139,11 +165,12 @@ define([ const flags = []; let p = start; for(; p < end; ++ p) { - const rawFlag = tokenKeyword(line[p]); + const token = line[p]; + const rawFlag = tokenKeyword(token); const flag = flagTypes[rawFlag]; if(flag) { if(flags.includes(flag)) { - throw new Error('Duplicate agent flag: ' + rawFlag); + throw makeError('Duplicate agent flag: ' + rawFlag, token); } flags.push(flag); } else { @@ -204,7 +231,7 @@ define([ const type = tokenKeyword(line[1]); if(TERMINATOR_TYPES.indexOf(type) === -1) { - throw new Error('Unknown termination: ' + joinLabel(line)); + throw makeError('Unknown termination "' + type + '"', line[1]); } meta.terminators = type; return true; @@ -274,14 +301,11 @@ define([ let skip = 2; skip = skipOver(line, skip, type.skip); const agents = readAgentList(line, skip, labelSep); - if( - agents.length < type.min || - (type.max !== null && agents.length > type.max) - ) { - throw new Error( - 'Invalid ' + mode.mode + - ': ' + joinLabel(line) - ); + if(agents.length < type.min) { + throw makeError('Too few agents for ' + mode.mode, line[0]); + } + if(type.max !== null && agents.length > type.max) { + throw makeError('Too many agents for ' + mode.mode, line[0]); } return { type: type.type, @@ -343,9 +367,13 @@ define([ } } if(!stage) { - throw new Error('Unrecognised command: ' + joinLabel(line)); + throw makeError( + 'Unrecognised command: ' + joinLabel(line), + line[0] + ); } if(typeof stage === 'object') { + stage.ln = line[0].b.ln; stages.push(stage); } } diff --git a/scripts/sequence/Parser_spec.js b/scripts/sequence/Parser_spec.js index c7254de..8f2cdf7 100644 --- a/scripts/sequence/Parser_spec.js +++ b/scripts/sequence/Parser_spec.js @@ -4,7 +4,43 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { const parser = new Parser(); const PARSED = { + blockBegin: ({ + ln = jasmine.anything(), + mode = jasmine.anything(), + label = jasmine.anything(), + } = {}) => { + return { + type: 'block begin', + ln, + mode, + label, + }; + }, + + blockSplit: ({ + ln = jasmine.anything(), + mode = jasmine.anything(), + label = jasmine.anything(), + } = {}) => { + return { + type: 'block split', + ln, + mode, + label, + }; + }, + + blockEnd: ({ + ln = jasmine.anything(), + } = {}) => { + return { + type: 'block end', + ln, + }; + }, + connect: (agentNames, { + ln = jasmine.anything(), line = jasmine.anything(), left = jasmine.anything(), right = jasmine.anything(), @@ -12,6 +48,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { } = {}) => { return { type: 'connect', + ln, agents: agentNames.map((name) => ({ name, alias: '', @@ -77,7 +114,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { it('propagates aliases', () => { const parsed = parser.parse('define Foo Bar as A B'); expect(parsed.stages).toEqual([ - {type: 'agent define', agents: [ + {type: 'agent define', ln: jasmine.anything(), agents: [ {name: 'Foo Bar', alias: 'A B', flags: []}, ]}, ]); @@ -102,6 +139,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { expect(parsed.stages).toEqual([ { type: 'connect', + ln: jasmine.anything(), agents: [ {name: 'A', alias: '', flags: ['start']}, {name: 'B', alias: '', flags: [ @@ -117,12 +155,18 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { }); it('rejects duplicate flags', () => { - expect(() => parser.parse('A -> +*+B')).toThrow(); - expect(() => parser.parse('A -> **B')).toThrow(); + expect(() => parser.parse('A -> +*+B')).toThrow(new Error( + 'Duplicate agent flag: + at line 1, character 7' + )); + expect(() => parser.parse('A -> **B')).toThrow(new Error( + 'Duplicate agent flag: * at line 1, character 6' + )); }); it('rejects missing agent names', () => { - expect(() => parser.parse('A -> +')).toThrow(); + expect(() => parser.parse('A -> +')).toThrow(new Error( + 'Missing agent name at line 1, character 6' + )); }); it('converts multiple entries', () => { @@ -141,6 +185,14 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { ]); }); + it('stores line numbers', () => { + const parsed = parser.parse('A -> B\nB -> A'); + expect(parsed.stages).toEqual([ + PARSED.connect(['A', 'B'], {ln: 0}), + PARSED.connect(['B', 'A'], {ln: 1}), + ]); + }); + it('recognises all types of connection', () => { const parsed = parser.parse( 'A -> B\n' + @@ -215,6 +267,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { const parsed = parser.parse('note over A: hello there'); expect(parsed.stages).toEqual([{ type: 'note over', + ln: jasmine.anything(), agents: [{name: 'A', alias: '', flags: []}], mode: 'note', label: 'hello there', @@ -232,30 +285,35 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { expect(parsed.stages).toEqual([ { type: 'note left', + ln: jasmine.anything(), agents: [{name: 'A', alias: '', flags: []}], mode: 'note', label: 'hello there', }, { type: 'note left', + ln: jasmine.anything(), agents: [{name: 'A', alias: '', flags: []}], mode: 'note', label: 'hello there', }, { type: 'note right', + ln: jasmine.anything(), agents: [{name: 'A', alias: '', flags: []}], mode: 'note', label: 'hello there', }, { type: 'note right', + ln: jasmine.anything(), agents: [{name: 'A', alias: '', flags: []}], mode: 'note', label: 'hello there', }, { type: 'note between', + ln: jasmine.anything(), agents: [ {name: 'A', alias: '', flags: []}, {name: 'B', alias: '', flags: []}, @@ -270,6 +328,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { const parsed = parser.parse('note over A B, C D: hi'); expect(parsed.stages).toEqual([{ type: 'note over', + ln: jasmine.anything(), agents: [ {name: 'A B', alias: '', flags: []}, {name: 'C D', alias: '', flags: []}, @@ -280,13 +339,16 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { }); it('rejects note between for a single agent', () => { - expect(() => parser.parse('note between A: hi')).toThrow(); + expect(() => parser.parse('note between A: hi')).toThrow(new Error( + 'Too few agents for note at line 1, character 0' + )); }); it('converts state', () => { const parsed = parser.parse('state over A: doing stuff'); expect(parsed.stages).toEqual([{ type: 'note over', + ln: jasmine.anything(), agents: [{name: 'A', alias: '', flags: []}], mode: 'state', label: 'doing stuff', @@ -294,13 +356,16 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { }); it('rejects multiple agents for state', () => { - expect(() => parser.parse('state over A, B: hi')).toThrow(); + expect(() => parser.parse('state over A, B: hi')).toThrow(new Error( + 'Too many agents for state at line 1, character 0' + )); }); it('converts text blocks', () => { const parsed = parser.parse('text right of A: doing stuff'); expect(parsed.stages).toEqual([{ type: 'note right', + ln: jasmine.anything(), agents: [{name: 'A', alias: '', flags: []}], mode: 'text', label: 'doing stuff', @@ -316,6 +381,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { expect(parsed.stages).toEqual([ { type: 'agent define', + ln: jasmine.anything(), agents: [ {name: 'A', alias: '', flags: []}, {name: 'B', alias: '', flags: []}, @@ -323,6 +389,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { }, { type: 'agent begin', + ln: jasmine.anything(), agents: [ {name: 'A', alias: '', flags: []}, {name: 'B', alias: '', flags: []}, @@ -331,6 +398,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { }, { type: 'agent end', + ln: jasmine.anything(), agents: [ {name: 'A', alias: '', flags: []}, {name: 'B', alias: '', flags: []}, @@ -344,6 +412,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { const parsed = parser.parse('abc:'); expect(parsed.stages).toEqual([{ type: 'mark', + ln: jasmine.anything(), name: 'abc', }]); }); @@ -352,6 +421,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { const parsed = parser.parse('simultaneously:'); expect(parsed.stages).toEqual([{ type: 'async', + ln: jasmine.anything(), target: '', }]); }); @@ -360,6 +430,7 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { const parsed = parser.parse('simultaneously with abc:'); expect(parsed.stages).toEqual([{ type: 'async', + ln: jasmine.anything(), target: 'abc', }]); }); @@ -376,21 +447,21 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { 'end\n' ); expect(parsed.stages).toEqual([ - {type: 'block begin', mode: 'if', label: 'something happens'}, + PARSED.blockBegin({mode: 'if', label: 'something happens'}), PARSED.connect(['A', 'B']), - {type: 'block split', mode: 'else', label: 'something else'}, + PARSED.blockSplit({mode: 'else', label: 'something else'}), PARSED.connect(['A', 'C']), PARSED.connect(['C', 'B']), - {type: 'block split', mode: 'else', label: ''}, + PARSED.blockSplit({mode: 'else', label: ''}), PARSED.connect(['A', 'D']), - {type: 'block end'}, + PARSED.blockEnd(), ]); }); it('converts loop blocks', () => { const parsed = parser.parse('repeat until something'); expect(parsed.stages).toEqual([ - {type: 'block begin', mode: 'repeat', label: 'until something'}, + PARSED.blockBegin({mode: 'repeat', label: 'until something'}), ]); }); @@ -399,7 +470,9 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { }); it('rejects invalid inputs', () => { - expect(() => parser.parse('huh')).toThrow(); + expect(() => parser.parse('huh')).toThrow(new Error( + 'Unrecognised command: huh at line 1, character 0' + )); }); it('rejects partial links', () => { @@ -409,7 +482,9 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { }); it('rejects invalid terminators', () => { - expect(() => parser.parse('terminators foo')).toThrow(); + expect(() => parser.parse('terminators foo')).toThrow(new Error( + 'Unknown termination "foo" at line 1, character 12' + )); }); it('rejects malformed notes', () => { @@ -417,7 +492,9 @@ defineDescribe('Sequence Parser', ['./Parser'], (Parser) => { }); it('rejects malformed block commands', () => { - expect(() => parser.parse('else nope foo')).toThrow(); + expect(() => parser.parse('else nope foo')).toThrow(new Error( + 'Invalid block command; expected "if" at line 1, character 5' + )); }); it('rejects invalid notes', () => { diff --git a/scripts/sequence/Tokeniser.js b/scripts/sequence/Tokeniser.js index 7105d12..e9a9779 100644 --- a/scripts/sequence/Tokeniser.js +++ b/scripts/sequence/Tokeniser.js @@ -104,30 +104,93 @@ define(['./CodeMirrorMode'], (CMMode) => { } } + function copyPos(pos) { + return {i: pos.i, ln: pos.ln, ch: pos.ch}; + } + + function advancePos(pos, src, steps) { + for(let i = 0; i < steps; ++ i) { + ++ pos.ch; + if(src[pos.i + i] === '\n') { + ++ pos.ln; + pos.ch = 0; + } + } + pos.i += steps; + } + + class TokenState { + constructor(src) { + this.src = src; + this.block = null; + this.token = null; + this.pos = {i: 0, ln: 0, ch: 0}; + this.reset(); + } + + isOver() { + return this.pos.i > this.src.length; + } + + reset() { + this.token = {s: '', v: '', q: false, b: null, e: null}; + this.block = null; + } + + beginToken(advance) { + this.block = advance.newBlock; + Object.assign(this.token, this.block.baseToken); + this.token.b = copyPos(this.pos); + } + + endToken() { + let token = null; + if(!this.block.omit) { + this.token.e = copyPos(this.pos); + token = this.token; + } + this.reset(); + return token; + } + + advance() { + const advance = tokAdvance(this.src, this.pos.i, this.block); + + if(advance.newBlock) { + this.beginToken(advance); + } + + this.token.s += advance.appendSpace; + this.token.v += advance.appendValue; + advancePos(this.pos, this.src, advance.skip); + + if(advance.end) { + return this.endToken(); + } else { + return null; + } + } + } + + function posStr(pos) { + return 'line ' + (pos.ln + 1) + ', character ' + pos.ch; + } + return class Tokeniser { tokenise(src) { const tokens = []; - let block = null; - let current = {s: '', v: '', q: false}; - for(let i = 0; i <= src.length;) { - const advance = tokAdvance(src, i, block); - if(advance.newBlock) { - block = advance.newBlock; - Object.assign(current, block.baseToken); - } - current.s += advance.appendSpace; - current.v += advance.appendValue; - i += advance.skip; - if(advance.end) { - if(!block.omit) { - tokens.push(current); - } - current = {s: '', v: '', q: false}; - block = null; + const state = new TokenState(src); + while(!state.isOver()) { + const token = state.advance(); + if(token) { + tokens.push(token); } } - if(block) { - throw new Error('Unterminated block'); + if(state.block) { + throw new Error( + 'Unterminated literal (began at ' + + posStr(state.token.b) + ')' + ); } return tokens; } diff --git a/scripts/sequence/Tokeniser_spec.js b/scripts/sequence/Tokeniser_spec.js index 39e6e3c..b1d67e7 100644 --- a/scripts/sequence/Tokeniser_spec.js +++ b/scripts/sequence/Tokeniser_spec.js @@ -4,14 +4,24 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const tokeniser = new Tokeniser(); describe('.tokenise', () => { + function token({ + s = jasmine.anything(), // spacing + v = jasmine.anything(), // value + q = jasmine.anything(), // isQuote? + b = jasmine.anything(), // begin position + e = jasmine.anything(), // end position + }) { + return {s, v, q, b, e}; + } + it('converts the source into atomic tokens', () => { const input = 'foo bar -> baz'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: ' ', v: 'bar', q: false}, - {s: ' ', v: '->', q: false}, - {s: ' ', v: 'baz', q: false}, + token({s: '', v: 'foo'}), + token({s: ' ', v: 'bar'}), + token({s: ' ', v: '->'}), + token({s: ' ', v: 'baz'}), ]); }); @@ -19,10 +29,21 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = 'foo bar->baz'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: ' ', v: 'bar', q: false}, - {s: '', v: '->', q: false}, - {s: '', v: 'baz', q: false}, + token({s: '', v: 'foo'}), + token({s: ' ', v: 'bar'}), + token({s: '', v: '->'}), + token({s: '', v: 'baz'}), + ]); + }); + + it('stores character numbers', () => { + const input = 'foo bar -> baz'; + const tokens = tokeniser.tokenise(input); + expect(tokens).toEqual([ + token({b: {i: 0, ln: 0, ch: 0}, e: {i: 3, ln: 0, ch: 3}}), + token({b: {i: 4, ln: 0, ch: 4}, e: {i: 7, ln: 0, ch: 7}}), + token({b: {i: 8, ln: 0, ch: 8}, e: {i: 10, ln: 0, ch: 10}}), + token({b: {i: 11, ln: 0, ch: 11}, e: {i: 14, ln: 0, ch: 14}}), ]); }); @@ -30,10 +51,21 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = 'foo bar\nbaz'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: ' ', v: 'bar', q: false}, - {s: '', v: '\n', q: false}, - {s: '', v: 'baz', q: false}, + token({s: '', v: 'foo'}), + token({s: ' ', v: 'bar'}), + token({s: '', v: '\n'}), + token({s: '', v: 'baz'}), + ]); + }); + + it('stores line numbers', () => { + const input = 'foo bar\nbaz'; + const tokens = tokeniser.tokenise(input); + expect(tokens).toEqual([ + token({b: {i: 0, ln: 0, ch: 0}, e: {i: 3, ln: 0, ch: 3}}), + token({b: {i: 4, ln: 0, ch: 4}, e: {i: 7, ln: 0, ch: 7}}), + token({b: {i: 7, ln: 0, ch: 7}, e: {i: 8, ln: 1, ch: 0}}), + token({b: {i: 8, ln: 1, ch: 0}, e: {i: 11, ln: 1, ch: 3}}), ]); }); @@ -41,9 +73,9 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = 'foo "\n" baz'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: ' ', v: '\n', q: true}, - {s: ' ', v: 'baz', q: false}, + token({s: '', v: 'foo', q: false}), + token({s: ' ', v: '\n', q: true}), + token({s: ' ', v: 'baz', q: false}), ]); }); @@ -51,10 +83,10 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = ' foo \t bar\t\n baz'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: ' ', v: 'foo', q: false}, - {s: ' \t ', v: 'bar', q: false}, - {s: '\t', v: '\n', q: false}, - {s: ' ', v: 'baz', q: false}, + token({s: ' ', v: 'foo'}), + token({s: ' \t ', v: 'bar'}), + token({s: '\t', v: '\n'}), + token({s: ' ', v: 'baz'}), ]); }); @@ -62,9 +94,17 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = 'foo "zig zag" \'abc def\''; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: ' ', v: 'zig zag', q: true}, - {s: ' ', v: 'abc def', q: true}, + token({s: '', v: 'foo', q: false}), + token({s: ' ', v: 'zig zag', q: true}), + token({s: ' ', v: 'abc def', q: true}), + ]); + }); + + it('stores character positions around quoted strings', () => { + const input = '"foo bar"'; + const tokens = tokeniser.tokenise(input); + expect(tokens).toEqual([ + token({b: {i: 0, ln: 0, ch: 0}, e: {i: 9, ln: 0, ch: 9}}), ]); }); @@ -72,9 +112,9 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = 'foo # bar baz\nzig'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: '', v: '\n', q: false}, - {s: '', v: 'zig', q: false}, + token({s: '', v: 'foo'}), + token({s: '', v: '\n'}), + token({s: '', v: 'zig'}), ]); }); @@ -82,9 +122,9 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = 'foo # bar "\'baz\nzig'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: '', v: '\n', q: false}, - {s: '', v: 'zig', q: false}, + token({s: '', v: 'foo'}), + token({s: '', v: '\n'}), + token({s: '', v: 'zig'}), ]); }); @@ -92,8 +132,8 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = 'foo "zig\\" zag\\n"'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: ' ', v: 'zig" zag\n', q: true}, + token({s: '', v: 'foo', q: false}), + token({s: ' ', v: 'zig" zag\n', q: true}), ]); }); @@ -101,13 +141,27 @@ defineDescribe('Sequence Tokeniser', ['./Tokeniser'], (Tokeniser) => { const input = 'foo " zig\n zag "'; const tokens = tokeniser.tokenise(input); expect(tokens).toEqual([ - {s: '', v: 'foo', q: false}, - {s: ' ', v: ' zig\n zag ', q: true}, + token({s: '', v: 'foo', q: false}), + token({s: ' ', v: ' zig\n zag ', q: true}), + ]); + }); + + it('calculates line numbers consistently within quotes', () => { + const input = 'foo\nbar "zig\nzag\na" b'; + const tokens = tokeniser.tokenise(input); + expect(tokens).toEqual([ + token({b: {i: 0, ln: 0, ch: 0}, e: {i: 3, ln: 0, ch: 3}}), + token({b: {i: 3, ln: 0, ch: 3}, e: {i: 4, ln: 1, ch: 0}}), + token({b: {i: 4, ln: 1, ch: 0}, e: {i: 7, ln: 1, ch: 3}}), + token({b: {i: 8, ln: 1, ch: 4}, e: {i: 19, ln: 3, ch: 2}}), + token({b: {i: 20, ln: 3, ch: 3}, e: {i: 21, ln: 3, ch: 4}}), ]); }); it('rejects unterminated quoted values', () => { - expect(() => tokeniser.tokenise('"nope')).toThrow(); + expect(() => tokeniser.tokenise('"nope')).toThrow(new Error( + 'Unterminated literal (began at line 1, character 0)' + )); }); });