describe('Ext.parse.Tokenizer', function () {
    var tokenizer,
        OPERATORS = {
            '+': { type: 'operator', value: '+', is: { operator: true, plus: true } },
            '-': { type: 'operator', value: '-', is: { operator: true, minus: true } },
            '*': { type: 'operator', value: '*', is: { operator: true, multiply: true } },
            '/': { type: 'operator', value: '/', is: { operator: true, divide: true } },
            '!': { type: 'operator', value: '!', is: { operator: true, bang: true } },
            ',': { type: 'operator', value: ',', is: { operator: true, comma: true } },
            ':': { type: 'operator', value: ':', is: { operator: true, colon: true } },
            '[': { type: 'operator', value: '[', is: { operator: true, arrayOpen: true } },
            ']': { type: 'operator', value: ']', is: { operator: true, arrayClose: true } },
            '{': { type: 'operator', value: '{', is: { operator: true, curlyOpen: true } },
            '}': { type: 'operator', value: '}', is: { operator: true, curlyClose: true } },
            '(': { type: 'operator', value: '(', is: { operator: true, parenOpen: true } },
            ')': { type: 'operator', value: ')', is: { operator: true, parenClose: true } }
        };

    beforeEach(function () {
        tokenizer = Ext.parse.Tokenizer.fly();
    });
    afterEach(function () {
        tokenizer.release();
        tokenizer = null;
    });

    function tokenize (tokens, text) {
        var ret = [];
        var tok;

        if (!text) {
            text = tokens;
            tokens = tokenizer;
        }

        tokens.reset(text);

        do {
            ret.push(tok = tokens.next());
        } while (tok && !tok.is.error);

        return ret;
    }

    describe('fly', function () {
        it('should return the same instance', function () {
            var f0 = Ext.parse.Tokenizer.fly();
            f0.release();

            var f1 = Ext.parse.Tokenizer.fly();
            expect(f1).toBe(f0);
            f1.release();

            var f2 = Ext.parse.Tokenizer.fly();
            expect(f2).toBe(f0);
        });
    });

    describe('peek', function () {
        var t0, t1, t2, t3, t4, t5, t6, t7;

        beforeEach(function () {
            if (!t0) {
                tokenizer.reset('abc 123');

                t0 = tokenizer.peek();
                t1 = tokenizer.peek();
                t2 = tokenizer.next();

                t3 = tokenizer.peek();
                t4 = tokenizer.peek();
                t5 = tokenizer.next();

                t6 = tokenizer.peek();
                t7 = tokenizer.next();
            }
        });

        it('should return the first token', function () {
            expect(t0).toEqual({
                type: 'ident',
                is: {
                    ident: true
                },
                value: 'abc'
            });
        });

        it('should return the same first token', function () {
            expect(t1).toBe(t0);
        });

        it('should return and consume the same first token', function () {
            expect(t2).toBe(t0);
        });

        it('should return the second token', function () {
            expect(t3).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 123
            });
        });

        it('should return the same second token', function () {
            expect(t4).toBe(t3);
        });

        it('should return and consume the same second token', function () {
            expect(t5).toBe(t3);
        });

        it('should detect end of tokens via peek', function () {
            expect(t6).toBe(null);
        });

        it('should detect end of tokens via next', function () {
            expect(t7).toBe(null);
        });
    });

    describe('strings', function () {
        it('should handle simple strings', function () {
            tokenizer.reset('"String 1" "String 2"');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t3 = tokenizer.next();
            var t4 = tokenizer.next();

            expect(t0).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    string: true
                },
                value: 'String 1'
            });

            expect(t1).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    string: true
                },
                value: 'String 2'
            });

            expect(t3).toBe(null);
            expect(t4).toBe(null);
        });

        it('should handle both types of quotes and escapes', function () {
            tokenizer.reset('\'String 1"\' "\'String\\\\\\\" 2\'"');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t3 = tokenizer.next();

            expect(t0).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    string: true
                },
                value: 'String 1"'
            });

            expect(t1).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    string: true
                },
                value: '\'String\\" 2\''
            });

            expect(t3).toBe(null);
        });
    });

    describe('booleans', function () {
        it('should handle true', function () {
            tokenizer.reset('true true');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    boolean: true
                },
                value: true
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should handle false', function () {
            tokenizer.reset('  false  \t false  ');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    boolean: true
                },
                value: false
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should handle mixed values', function () {
            tokenizer.reset('\tfalse\ttrue\t');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    boolean: true
                },
                value: false
            });

            expect(t1).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    boolean: true
                },
                value: true
            });

            expect(t2).toBe(null);
        });
    });

    describe('identifiers', function () {
        it('should handle simple identifiers', function () {
            tokenizer.reset('  foo  \tbar');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'ident',
                is: {
                    ident: true
                },
                value: 'foo'
            });

            expect(t1).toEqual({
                type: 'ident',
                is: {
                    ident: true
                },
                value: 'bar'
            });

            expect(t2).toBe(null);
        });

        it('should handle dotpath identifiers', function () {
            tokenizer.reset('foo.bar\t bar.baz.zip ');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'ident',
                is: {
                    ident: true
                },
                value: 'foo.bar'
            });

            expect(t1).toEqual({
                type: 'ident',
                is: {
                    ident: true
                },
                value: 'bar.baz.zip'
            });

            expect(t2).toBe(null);
        });
    });

    describe('null', function () {
        it('should handle null', function () {
            tokenizer.reset('  null  \tnull');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    nil: true
                },
                value: null
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });
    });

    describe('numbers', function () {
        it('should be able to parse integers', function () {
            tokenizer.reset('  427  \t23');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 427
            });

            expect(t1).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 23
            });

            expect(t2).toBe(null);
        });

        it('should be able to parse signed integers', function () {
            tokenizer.reset('  +427  \t-23 21');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();
            var t3 = tokenizer.next();
            var t4 = tokenizer.next();
            var t5 = tokenizer.next();

            expect(t0).toEqual(OPERATORS['+']);
            expect(t1).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 427
            });

            expect(t2).toEqual(OPERATORS['-']);
            expect(t3).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 23
            });

            expect(t4).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 21
            });

            expect(t5).toBe(null);
        });

        it('should be able to parse decimals', function () {
            tokenizer.reset('  +.427  \t-23.234 2.1');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();
            var t3 = tokenizer.next();
            var t4 = tokenizer.next();
            var t5 = tokenizer.next();

            expect(t0).toEqual(OPERATORS['+']);
            expect(t1).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 0.427
            });

            expect(t2).toEqual(OPERATORS['-']);
            expect(t3).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 23.234
            });

            expect(t4).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 2.1
            });

            expect(t5).toBe(null);
        });

        it('should be able to parse exponentials', function () {
            tokenizer.reset('  +.42e7  \t-23.234e+2 2.1e-21');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();
            var t3 = tokenizer.next();
            var t4 = tokenizer.next();
            var t5 = tokenizer.next();

            expect(t0).toEqual(OPERATORS['+']);
            expect(t1).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 0.42e7
            });

            expect(t2).toEqual(OPERATORS['-']);
            expect(t3).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 23.234e2
            });

            expect(t4).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 2.1e-21
            });

            expect(t5).toBe(null);
        });
    });

    describe('operators', function () {
        it('should be able to parse exclamation', function () {
            tokenizer.reset(' ! ! ');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: '!',
                is: {
                    operator: true,
                    bang: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should be able to parse comma', function () {
            tokenizer.reset(',,');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: ',',
                is: {
                    operator: true,
                    comma: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should be able to parse colon', function () {
            tokenizer.reset('\t:\t: ');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: ':',
                is: {
                    operator: true,
                    colon: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should be able to parse array open', function () {
            tokenizer.reset('[ [');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: '[',
                is: {
                    operator: true,
                    arrayOpen: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should be able to parse array close', function () {
            tokenizer.reset(' ]] ');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: ']',
                is: {
                    operator: true,
                    arrayClose: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should be able to parse brace open', function () {
            tokenizer.reset('\t  {{\t');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: '{',
                is: {
                    operator: true,
                    curlyOpen: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should be able to parse brace close', function () {
            tokenizer.reset(' }}');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: '}',
                is: {
                    operator: true,
                    curlyClose: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should be able to parse paren open', function () {
            tokenizer.reset(' ( (\t');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: '(',
                is: {
                    operator: true,
                    parenOpen: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });

        it('should be able to parse paren close', function () {
            tokenizer.reset('))');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'operator',
                value: ')',
                is: {
                    operator: true,
                    parenClose: true
                }
            });

            expect(t0).toBe(t1);  // should reuse the same token instance

            expect(t2).toBe(null);
        });
    });

    describe('custom operators', function () {
        var customTokenizer,
            tokens;

        beforeEach(function () {
            customTokenizer = new Ext.parse.Tokenizer({
                operators: {
                    '!': null,  // disable ! operator
                    '$': 'dollar'
                }
            });

            tokens = tokenize(customTokenizer, '$!');
        });

        it('should tokenize custom operator', function () {
            expect(tokens[0]).toEqual({
                type: 'operator',
                value: '$',
                is: {
                    operator: true,
                    dollar: true
                }
            });
        });

        it('should disable standard operator', function () {
            var tok = tokens[1];

            expect(tok instanceof Error).toBe(true);
            expect(tok.type).toBe('error');
            expect(tok.is.error).toBe(true);
            expect(tok.at).toBe(1);
        });
    });

    describe('multi-character operators', function () {
        var multiOpTokenizer,
            tokens;

        beforeEach(function () {
            multiOpTokenizer = new Ext.parse.Tokenizer({
                operators: {
                    '!=': 'ne',
                    '==': 'eq',
                    '<=': 'le',
                    '<':  'lt',
                    '>':  'gt',
                    '>=': 'ge',
                    '=': 'assign',
                    '===': 'seq',
                    '!==': 'sne'
                }
            });

            tokens = tokenize(multiOpTokenizer, '< <= > >= \t=\t!= == === !==');
        });

        it('should tokenize less-than operator', function () {
            expect(tokens[0]).toEqual({
                type: 'operator',
                value: '<',
                is: {
                    operator: true,
                    lt: true
                }
            });
        });

        it('should tokenize less-than-or-equal operator', function () {
            expect(tokens[1]).toEqual({
                type: 'operator',
                value: '<=',
                is: {
                    operator: true,
                    le: true
                }
            });
        });

        it('should tokenize greater-than operator', function () {
            expect(tokens[2]).toEqual({
                type: 'operator',
                value: '>',
                is: {
                    operator: true,
                    gt: true
                }
            });
        });

        it('should tokenize greater-than-or-equal operator', function () {
            expect(tokens[3]).toEqual({
                type: 'operator',
                value: '>=',
                is: {
                    operator: true,
                    ge: true
                }
            });
        });

        it('should tokenize assignment operator', function () {
            expect(tokens[4]).toEqual({
                type: 'operator',
                value: '=',
                is: {
                    operator: true,
                    assign: true
                }
            });
        });

        it('should tokenize not-equals operator', function () {
            expect(tokens[5]).toEqual({
                type: 'operator',
                value: '!=',
                is: {
                    operator: true,
                    ne: true
                }
            });
        });

        it('should tokenize equals operator', function () {
            expect(tokens[6]).toEqual({
                type: 'operator',
                value: '==',
                is: {
                    operator: true,
                    eq: true
                }
            });
        });

        it('should tokenize strict equality operator', function () {
            expect(tokens[7]).toEqual({
                type: 'operator',
                value: '===',
                is: {
                    operator: true,
                    seq: true
                }
            });
        });

        it('should tokenize strict inequality operator', function () {
            expect(tokens[8]).toEqual({
                type: 'operator',
                value: '!==',
                is: {
                    operator: true,
                    sne: true
                }
            });
        });
    });

    describe('all the things', function () {
        var tokens;

        beforeEach(function () {
            if (tokens) {
                return;
            }

            tokens = tokenize('foo: bar ( ' +
                    '"\\"a\\\\b\'):\\"" , '+
                    '32,' +
                    '! zip.fiz:woot(true,null)'+
                '):ack(\'x\\\'"y\', 32e-21 , -3.14e0 )');
        });

        it('should parse token 0', function () {
            expect(tokens[0]).toEqual(
                { type: 'ident', is: { ident: true }, value: 'foo' }
            );
        });

        it('should parse token 1', function () {
            expect(tokens[1]).toEqual(OPERATORS[':']);
        });

        it('should parse token 2', function () {
            expect(tokens[2]).toEqual(
                { type: 'ident', is: { ident: true }, value: 'bar' }
            );
        });

        it('should parse token 3', function () {
            expect(tokens[3]).toEqual(OPERATORS['(']);
        });

        it('should parse token 4', function () {
            expect(tokens[4]).toEqual(
                { type: 'literal', is: { literal: true, string: true }, value: '"a\\b\'):"' }
            );
        });

        it('should parse token 5', function () {
            expect(tokens[5]).toEqual(OPERATORS[',']);
        });

        it('should parse token 6', function () {
            expect(tokens[6]).toEqual(
                { type: 'literal', is: { literal: true, number: true }, value: 32 }
            );
        });

        it('should parse token 7', function () {
            expect(tokens[7]).toEqual(OPERATORS[',']);
        });

        it('should parse token 8', function () {
            expect(tokens[8]).toEqual(OPERATORS['!']);
        });

        it('should parse token 9', function () {
            expect(tokens[9]).toEqual(
                { type: 'ident', is: { ident: true }, value: 'zip.fiz' }
            );
        });

        it('should parse token 10', function () {
            expect(tokens[10]).toEqual(OPERATORS[':']);
        });

        it('should parse token 11', function () {
            expect(tokens[11]).toEqual(
                { type: 'ident', is: { ident: true }, value: 'woot' }
            );
        });

        it('should parse token 12', function () {
            expect(tokens[12]).toEqual(OPERATORS['(']);
        });

        it('should parse token 13', function () {
            expect(tokens[13]).toEqual(
                { type: 'literal', is: { literal: true, boolean: true }, value: true }
            );
        });

        it('should parse token 14', function () {
            expect(tokens[14]).toEqual(OPERATORS[',']);
        });

        it('should parse token 15', function () {
            expect(tokens[15]).toEqual(
                { type: 'literal', is: { literal: true, nil: true }, value: null }
            );
        });

        it('should parse token 16', function () {
            expect(tokens[16]).toEqual(OPERATORS[')']);
        });

        it('should parse token 17', function () {
            expect(tokens[17]).toEqual(OPERATORS[')']);
        });

        it('should parse token 18', function () {
            expect(tokens[18]).toEqual(OPERATORS[':']);
        });

        it('should parse token 19', function () {
            expect(tokens[19]).toEqual(
                { type: 'ident', is: { ident: true }, value: 'ack' }
            );
        });

        it('should parse token 20', function () {
            expect(tokens[20]).toEqual(OPERATORS['(']);
        });

        it('should parse token 21', function () {
            expect(tokens[21]).toEqual(
                { type: 'literal', is: { literal: true, string: true }, value: 'x\'"y' }
            );
        });

        it('should parse token 22', function () {
            expect(tokens[22]).toEqual(OPERATORS[',']);
        });

        it('should parse token 23', function () {
            expect(tokens[23]).toEqual(
                { type: 'literal', is: { literal: true, number: true }, value: 32e-21 }
            );
        });

        it('should parse token 24', function () {
            expect(tokens[24]).toEqual(OPERATORS[',']);
        });

        it('should parse token 25', function () {
            expect(tokens[24]).toEqual(OPERATORS[',']);
        });

        it('should parse token 26', function () {
            expect(tokens[26]).toEqual(
                { type: 'literal', is: { literal: true, number: true }, value: 3.14 }
            );
        });

        it('should parse token 27', function () {
            expect(tokens[27]).toEqual(OPERATORS[')']);
        });

        it('should parse token 28', function () {
            expect(tokens[28]).toBe(null);
        });

        it('should parse all the tokens', function () {
            expect(tokens.length).toBe(29);
        });
    });

    describe('syntax errors', function () {
        it('should catch invalid characters', function () {
            tokenizer.reset('&');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0 instanceof Error).toBe(true);
            expect(t0.type).toBe('error');
            expect(t0.is.error).toBe(true);
            expect(t0.at).toBe(0);

            expect(t1).toBe(t0);  // should reuse the same token instance
            expect(t2).toBe(t0);  // never recovers from error
        });

        it('should catch invalid characters after valid ones', function () {
            tokenizer.reset('123&');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'literal',
                is: {
                    literal: true,
                    number: true
                },
                value: 123
            });

            expect(t1 instanceof Error).toBe(true);
            expect(t1.type).toBe('error');
            expect(t1.is.error).toBe(true);
            expect(t1.at).toBe(3);

            expect(t2).toBe(t1);  // never recovers from error
        });

        it('should handle errors after dotpath identifiers', function () {
            tokenizer.reset('foo.bar\t bar.baz.zip &');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();
            var t2 = tokenizer.next();

            expect(t0).toEqual({
                type: 'ident',
                is: {
                    ident: true
                },
                value: 'foo.bar'
            });

            expect(t1).toEqual({
                type: 'ident',
                is: {
                    ident: true
                },
                value: 'bar.baz.zip'
            });

            expect(t2 instanceof Error).toBe(true);
            expect(t2.type).toBe('error');
            expect(t2.is.error).toBe(true);
            expect(t2.at).toBe(21);
        });

        it('should report dotpaths with adjacent dots', function () {
            tokenizer.reset('foo..bar');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();

            expect(t0 instanceof Error).toBe(true);
            expect(t0.type).toBe('error');
            expect(t0.is.error).toBe(true);
            expect(t0.at).toBe(4);

            expect(t1).toBe(t0);
        });

        it('should report dotpaths that start with a dot', function () {
            tokenizer.reset(' .foo');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();

            expect(t0 instanceof Error).toBe(true);
            expect(t0.type).toBe('error');
            expect(t0.is.error).toBe(true);
            expect(t0.at).toBe(1);

            expect(t1).toBe(t0);
        });

        it('should report dotpaths that end with a dot', function () {
            tokenizer.reset('foo.');

            var t0 = tokenizer.next();
            var t1 = tokenizer.next();

            expect(t0 instanceof Error).toBe(true);
            expect(t0.type).toBe('error');
            expect(t0.is.error).toBe(true);
            expect(t0.at).toBe(3);

            expect(t1).toBe(t0);
        });
    });
});
