Ntp.Analyzer.Config.Compiler.Tokenizer Class Reference

Inheritance diagram for Ntp.Analyzer.Config.Compiler.Tokenizer:

Collaboration diagram for Ntp.Analyzer.Config.Compiler.Tokenizer:

Public Member Functions
	Tokenizer (string file)
void	Consume ()
	Consume white spaces, line breaks and comments. More...
void	ConsumeWhiteSpaces ()
	Consume white spaces and comments. More...
void	Dispose ()
Token	GetNext ()
Token	PeekNext ()

Public Attributes
int	ColumnNumber => reader.Position
int	LineNumber => reader.Line

Properties
Token	Token [get, private set]

Private Member Functions
	~Tokenizer ()
void	ConsumeComment ()
void	Dispose (bool disposing)
Token	GetNextToken ()
Token	ParseLiteral ()
Token	ParseQuotedIdent ()

Static Private Member Functions
static Token	ParseBoolean (string text)
static Token	ParseDigitValue (string text)

Private Attributes
bool	disposedValue
readonly ConfigFileReader	reader

Detailed Description

Definition at line 29 of file Tokenizer.cs.

Constructor & Destructor Documentation

Ntp.Analyzer.Config.Compiler.Tokenizer.Tokenizer ( string  file )

inline

Definition at line 31 of file Tokenizer.cs.

References Ntp.Analyzer.Config.Table.Token.Undefined.

        {
            reader = new ConfigFileReader(file);
            Token = Token.Undefined;
        }

Ntp.Analyzer.Config.Compiler.Tokenizer.~Tokenizer ( )

inlineprivate

Definition at line 300 of file Tokenizer.cs.

        {
            Dispose(false);
        }

Member Function Documentation

void Ntp.Analyzer.Config.Compiler.Tokenizer.Consume ( )

inline

Consume white spaces, line breaks and comments.

Definition at line 48 of file Tokenizer.cs.

References c, Ntp.Analyzer.Config.Table.Token.NewLine, and Ntp.Analyzer.Config.Table.Token.Whitespace.

        {
            bool pass = true;
            do
            {
                char c = reader.Peek();
                switch (c)
                {
                    case ' ':
                    case '\t':
                        reader.Read();
                        Token = Token.Whitespace;
                        break;
                    case '#':
                        ConsumeComment();
                        break;
                    case '\n':
                        reader.Read();
                        reader.NextLine();
                        Token = Token.NewLine;
                        break;
                    case '\r':
                        reader.Read();
                        if (reader.Peek() == '\n')
                            reader.Read();

                        reader.NextLine();
                        Token = Token.NewLine;
                        break;
                    default:
                        pass = false;
                        break;
                }
            } while (pass);
        }

void Ntp.Analyzer.Config.Compiler.Tokenizer.ConsumeComment ( )

inlineprivate

Definition at line 123 of file Tokenizer.cs.

References Ntp.Analyzer.Config.Table.Token.NewLine.

        {
            if (reader.Peek() != '#')
                return;

            reader.Read();
            while (reader.Peek() != '\r' && reader.Peek() != '\n' && reader.Peek() != '\0')
                reader.Read();

            switch (reader.Peek())
            {
                case '\n':
                    reader.Read();
                    reader.NextLine();
                    Token = Token.NewLine;
                    return;
                case '\r':
                    reader.Read();
                    if (reader.Peek() == '\n')
                        reader.Read();

                    reader.NextLine();
                    Token = Token.NewLine;
                    break;
            }
        }

void Ntp.Analyzer.Config.Compiler.Tokenizer.ConsumeWhiteSpaces ( )

inline

Consume white spaces and comments.

Definition at line 87 of file Tokenizer.cs.

References Ntp.Analyzer.Config.Table.Token.Whitespace.

        {
            while (reader.Peek() == ' ' || reader.Peek() == '\t')
            {
                Token = Token.Whitespace;
                if (reader.Peek() == '#')
                    ConsumeComment();
                else
                    reader.Read();
            }
        }

void Ntp.Analyzer.Config.Compiler.Tokenizer.Dispose ( bool  disposing )

inlineprivate

Definition at line 287 of file Tokenizer.cs.

        {
            if (disposedValue)
                return;

            if (disposing)
            {
                reader.Dispose();
            }

            disposedValue = true;
        }

void Ntp.Analyzer.Config.Compiler.Tokenizer.Dispose ( )

inline

Definition at line 305 of file Tokenizer.cs.

        {
            Dispose(true);
            GC.SuppressFinalize(this);
        }

Token Ntp.Analyzer.Config.Compiler.Tokenizer.GetNext ( )

inline

Definition at line 99 of file Tokenizer.cs.

        {
            Token = GetNextToken();
            return Token;
        }

Token Ntp.Analyzer.Config.Compiler.Tokenizer.GetNextToken ( )

inlineprivate

Definition at line 150 of file Tokenizer.cs.

References c, Ntp.Analyzer.Config.Table.Token.NewLine, and Ntp.Analyzer.Config.Table.Token.Whitespace.

        {
            while (reader.Peek() != '\0')
            {
                switch (reader.Peek())
                {
                    case '\n':
                        reader.Read();
                        reader.NextLine();
                        return Token.NewLine;
                    case '\r':
                        reader.Read();
                        if (reader.Peek() == '\n')
                            reader.Read();

                        reader.NextLine();
                        return Token.NewLine;
                }

                while (char.IsWhiteSpace(reader.Peek()))
                {
                    reader.Read();
                    return Token.Whitespace;
                }

                if (reader.Peek() == '\0')
                {
                    return new SymbolToken(Symbol.EndOfFile);
                }

                var c = reader.Peek();
                switch (c)
                {
                    case '{':
                        reader.Read();
                        return new SymbolToken(Symbol.OpeningBrace);
                    case '}':
                        reader.Read();
                        return new SymbolToken(Symbol.ClosingBrace);
                }

                var token = ParseQuotedIdent() ?? ParseLiteral();
                return token;
            }

            return new SymbolToken(Symbol.EndOfFile);
        }

static Token Ntp.Analyzer.Config.Compiler.Tokenizer.ParseBoolean ( string  text )

inlinestaticprivate

Definition at line 198 of file Tokenizer.cs.

        {
            switch (text.ToLower())
            {
                case "0":
                    return new SymbolToken(Symbol.Zero);
                case "1":
                    return new SymbolToken(Symbol.One);
                case "no":
                case "off":
                case "false":
                    return new SymbolToken(Symbol.BooleanFalse);
                case "on":
                case "yes":
                case "true":
                    return new SymbolToken(Symbol.BooleanTrue);
                default:
                    return null;
            }
        }

static Token Ntp.Analyzer.Config.Compiler.Tokenizer.ParseDigitValue ( string  text )

inlinestaticprivate

Definition at line 219 of file Tokenizer.cs.

References c.

        {
            int value;
            if (int.TryParse(text, out value))
            {
                switch (value)
                {
                    case 0:
                        return new SymbolToken(Symbol.Zero);
                    case 1:
                        return new SymbolToken(Symbol.One);
                    default:
                        return new IntegerToken(value);
                }
            }

            if (text.ToCharArray().Count(c => "0123456789.".Contains(c)) != text.Length)
                return null;

            if (text.ToCharArray().Count(c => c == '.') > 1)
                return null;

            double dvalue;
            return double.TryParse(text, out dvalue) ? new NumericToken(dvalue) : null;
        }

Token Ntp.Analyzer.Config.Compiler.Tokenizer.ParseLiteral ( )

inlineprivate

Definition at line 245 of file Tokenizer.cs.

References Ntp.Analyzer.Config.Table.Keyword.Keywords.

        {
            var text = new StringBuilder();
            while (!char.IsWhiteSpace(reader.Peek()) && reader.Peek() != '\r' && reader.Peek() != '\n' &&
                   reader.Peek() != '\0')
                text.Append(reader.Read());

            string tokenText = text.ToString();

            var token = ParseBoolean(tokenText) ?? ParseDigitValue(tokenText);
            if (token != null)
                return token;

            string search = tokenText.ToLower();
            var keyword = Keyword.Keywords.FirstOrDefault(k => k.LowerText == search);

            return keyword != null
                ? new KeywordToken(keyword, tokenText)
                : (Token) new LiteralToken(tokenText);
        }

Token Ntp.Analyzer.Config.Compiler.Tokenizer.ParseQuotedIdent ( )

inlineprivate

Definition at line 266 of file Tokenizer.cs.

        {
            if (reader.Peek() != '"')
                return null;

            reader.Read();

            var text = new StringBuilder();
            while (reader.Peek() != '\0' && reader.Peek() != '"')
                text.Append(reader.Read());

            if (reader.Peek() == '"')
                reader.Read();

            return new QuotedIdentToken(text.ToString());
        }

Token Ntp.Analyzer.Config.Compiler.Tokenizer.PeekNext ( )

inline

Definition at line 105 of file Tokenizer.cs.

References Ntp.Analyzer.Config.Table.Token.EndOfFile, Ntp.Analyzer.Config.Table.Token.NewLine, Ntp.Analyzer.Config.Table.Token.Undefined, and Ntp.Analyzer.Config.Table.Token.Whitespace.

        {
            if (reader.Peek() == '\0')
                return Token.EndOfFile;

            switch (reader.Peek())
            {
                case '\r':
                case '\n':
                    return Token.NewLine;
                case ' ':
                case '\t':
                    return Token.Whitespace;
                default:
                    return Token.Undefined;
            }
        }

Member Data Documentation

int Ntp.Analyzer.Config.Compiler.Tokenizer.ColumnNumber => reader.Position

Definition at line 43 of file Tokenizer.cs.

bool Ntp.Analyzer.Config.Compiler.Tokenizer.disposedValue

private

Definition at line 285 of file Tokenizer.cs.

int Ntp.Analyzer.Config.Compiler.Tokenizer.LineNumber => reader.Line

Definition at line 41 of file Tokenizer.cs.

readonly ConfigFileReader Ntp.Analyzer.Config.Compiler.Tokenizer.reader

private

Definition at line 37 of file Tokenizer.cs.

Property Documentation

Token Ntp.Analyzer.Config.Compiler.Tokenizer.Token

getprivate set

Definition at line 39 of file Tokenizer.cs.

---

The documentation for this class was generated from the following file:

• Ntp.Analyzer/Config/Compiler/Tokenizer.cs