Tokenizer.cs

Go to the documentation of this file.

// 
// Copyright (c) 2013-2017 Carsten Sonne Larsen <cs@innolan.net>
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

using System;
using System.Linq;
using System.Text;
using Ntp.Analyzer.Config.Table;

namespace Ntp.Analyzer.Config.Compiler
{
    public sealed class Tokenizer : IDisposable
    {
        public Tokenizer(string file)
        {
            reader = new ConfigFileReader(file);
            Token = Token.Undefined;
        }

        private readonly ConfigFileReader reader;

        public Token Token { get; private set; }

        public int LineNumber => reader.Line;

        public int ColumnNumber => reader.Position;

        public void Consume()
        {
            bool pass = true;
            do
            {
                char c = reader.Peek();
                switch (c)
                {
                    case ' ':
                    case '\t':
                        reader.Read();
                        Token = Token.Whitespace;
                        break;
                    case '#':
                        ConsumeComment();
                        break;
                    case '\n':
                        reader.Read();
                        reader.NextLine();
                        Token = Token.NewLine;
                        break;
                    case '\r':
                        reader.Read();
                        if (reader.Peek() == '\n')
                            reader.Read();

                        reader.NextLine();
                        Token = Token.NewLine;
                        break;
                    default:
                        pass = false;
                        break;
                }
            } while (pass);
        }

        public void ConsumeWhiteSpaces()
        {
            while (reader.Peek() == ' ' || reader.Peek() == '\t')
            {
                Token = Token.Whitespace;
                if (reader.Peek() == '#')
                    ConsumeComment();
                else
                    reader.Read();
            }
        }

        public Token GetNext()
        {
            Token = GetNextToken();
            return Token;
        }

        public Token PeekNext()
        {
            if (reader.Peek() == '\0')
                return Token.EndOfFile;

            switch (reader.Peek())
            {
                case '\r':
                case '\n':
                    return Token.NewLine;
                case ' ':
                case '\t':
                    return Token.Whitespace;
                default:
                    return Token.Undefined;
            }
        }

        private void ConsumeComment()
        {
            if (reader.Peek() != '#')
                return;

            reader.Read();
            while (reader.Peek() != '\r' && reader.Peek() != '\n' && reader.Peek() != '\0')
                reader.Read();

            switch (reader.Peek())
            {
                case '\n':
                    reader.Read();
                    reader.NextLine();
                    Token = Token.NewLine;
                    return;
                case '\r':
                    reader.Read();
                    if (reader.Peek() == '\n')
                        reader.Read();

                    reader.NextLine();
                    Token = Token.NewLine;
                    break;
            }
        }

        private Token GetNextToken()
        {
            while (reader.Peek() != '\0')
            {
                switch (reader.Peek())
                {
                    case '\n':
                        reader.Read();
                        reader.NextLine();
                        return Token.NewLine;
                    case '\r':
                        reader.Read();
                        if (reader.Peek() == '\n')
                            reader.Read();

                        reader.NextLine();
                        return Token.NewLine;
                }

                while (char.IsWhiteSpace(reader.Peek()))
                {
                    reader.Read();
                    return Token.Whitespace;
                }

                if (reader.Peek() == '\0')
                {
                    return new SymbolToken(Symbol.EndOfFile);
                }

                var c = reader.Peek();
                switch (c)
                {
                    case '{':
                        reader.Read();
                        return new SymbolToken(Symbol.OpeningBrace);
                    case '}':
                        reader.Read();
                        return new SymbolToken(Symbol.ClosingBrace);
                }

                var token = ParseQuotedIdent() ?? ParseLiteral();
                return token;
            }

            return new SymbolToken(Symbol.EndOfFile);
        }

        private static Token ParseBoolean(string text)
        {
            switch (text.ToLower())
            {
                case "0":
                    return new SymbolToken(Symbol.Zero);
                case "1":
                    return new SymbolToken(Symbol.One);
                case "no":
                case "off":
                case "false":
                    return new SymbolToken(Symbol.BooleanFalse);
                case "on":
                case "yes":
                case "true":
                    return new SymbolToken(Symbol.BooleanTrue);
                default:
                    return null;
            }
        }

        private static Token ParseDigitValue(string text)
        {
            int value;
            if (int.TryParse(text, out value))
            {
                switch (value)
                {
                    case 0:
                        return new SymbolToken(Symbol.Zero);
                    case 1:
                        return new SymbolToken(Symbol.One);
                    default:
                        return new IntegerToken(value);
                }
            }

            if (text.ToCharArray().Count(c => "0123456789.".Contains(c)) != text.Length)
                return null;

            if (text.ToCharArray().Count(c => c == '.') > 1)
                return null;

            double dvalue;
            return double.TryParse(text, out dvalue) ? new NumericToken(dvalue) : null;
        }

        private Token ParseLiteral()
        {
            var text = new StringBuilder();
            while (!char.IsWhiteSpace(reader.Peek()) && reader.Peek() != '\r' && reader.Peek() != '\n' &&
                   reader.Peek() != '\0')
                text.Append(reader.Read());

            string tokenText = text.ToString();

            var token = ParseBoolean(tokenText) ?? ParseDigitValue(tokenText);
            if (token != null)
                return token;

            string search = tokenText.ToLower();
            var keyword = Keyword.Keywords.FirstOrDefault(k => k.LowerText == search);

            return keyword != null
                ? new KeywordToken(keyword, tokenText)
                : (Token) new LiteralToken(tokenText);
        }

        private Token ParseQuotedIdent()
        {
            if (reader.Peek() != '"')
                return null;

            reader.Read();

            var text = new StringBuilder();
            while (reader.Peek() != '\0' && reader.Peek() != '"')
                text.Append(reader.Read());

            if (reader.Peek() == '"')
                reader.Read();

            return new QuotedIdentToken(text.ToString());
        }

        #region IDisposable Support

        private bool disposedValue;

        private void Dispose(bool disposing)
        {
            if (disposedValue)
                return;

            if (disposing)
            {
                reader.Dispose();
            }

            disposedValue = true;
        }

        ~Tokenizer()
        {
            Dispose(false);
        }

        public void Dispose()
        {
            Dispose(true);
            GC.SuppressFinalize(this);
        }

        #endregion
    }
}