llvm-tutorial/source/Lexer.cpp

// Standard includes
#include <cctype>
#include <cstdio>

// Local includes
#include "Lexer.h"

/// gettok - Return the next token from standard input.
namespace lexer{

std::string LexerObjects::IdentifierStr; // Filled in if tok_identifier
double LexerObjects::NumVal;

std::string getTokName(int Tok) {
  switch (Tok) {
  case tok_eof:
    return "eof";
  case tok_def:
    return "def";
  case tok_extern:
    return "extern";
  case tok_identifier:
    return "identifier";
  case tok_number:
    return "number";
  case tok_if:
    return "if";
  case tok_then:
    return "then";
  case tok_else:
    return "else";
  case tok_for:
    return "for";
  case tok_in:
    return "in";
  case tok_binary:
    return "binary";
  case tok_unary:
    return "unary";
  case tok_var:
    return "var";
  }
  return std::string(1, (char)Tok);
}

int gettok() {
  static int LastChar = ' ';

  // Skip any whitespace.
  while (isspace(LastChar))
  LastChar = getchar();

  if (isalpha(LastChar)) { // identifier: [a-zA-Z][a-zA-Z0-9]*
    LexerObjects::IdentifierStr = LastChar;
    while (isalnum((LastChar = getchar())))
      LexerObjects::IdentifierStr += LastChar;

    if (LexerObjects::IdentifierStr == "def")
      return tok_def;
    if (LexerObjects::IdentifierStr == "extern")
      return tok_extern;
    if (LexerObjects::IdentifierStr == "if")
      return tok_if;
    if (LexerObjects::IdentifierStr == "then")
      return tok_then;
    if (LexerObjects::IdentifierStr == "else")
      return tok_else;
    if (LexerObjects::IdentifierStr == "for")
      return tok_for;
    if (LexerObjects::IdentifierStr == "in")
      return tok_in;
    if (LexerObjects::IdentifierStr == "binary")
      return tok_binary;
    if (LexerObjects::IdentifierStr == "unary")
      return tok_unary;
    if (LexerObjects::IdentifierStr == "var")
      return tok_var;
    return tok_identifier;
  }

  if (isdigit(LastChar) || LastChar == '.') { // Number: [0-9.]+
    std::string NumStr;
    do {
      NumStr += LastChar;
      LastChar = getchar();
    } while (isdigit(LastChar) || LastChar == '.');

    LexerObjects::NumVal = strtod(NumStr.c_str(), nullptr);
    return tok_number;
  }

  if (LastChar == '#') {
    // Comment until end of line.
    do
    LastChar = getchar();
    while (LastChar != EOF && LastChar != '\n' && LastChar != '\r');

    if (LastChar != EOF)
    return gettok();
  }

  // Check for end of file.  Don't eat the EOF.
  if (LastChar == EOF)
  return tok_eof;

  // Otherwise, just return the character as its ascii value.
  int ThisChar = LastChar;
  LastChar = getchar();
  return ThisChar;
}
}