tokenizer.cpp

#include "tokenizer.h"
#include <cctype>
#include <stdexcept>

static const std::vector<std::string> FUNCTIONS = {
    "sqrt", "sin", "cos", "tan",
    "log", "log2", "ln",
    "abs", "floor", "ceil"
};

static bool isFunction(const std::string& word) {
    for (const auto& f : FUNCTIONS)
        if (f == word) return true;
    return false;
}

std::vector<Token> Tokenizer::tokenize(const std::string& expr) {
    input = expr;
    pos = 0;
    std::vector<Token> tokens;

    while (!isAtEnd()) {
        skipWhitespace();
        if (isAtEnd()) break;

        char c = peek();

        if (std::isdigit(c) || (c == '.' && std::isdigit(input[pos + 1]))) {
            tokens.push_back(readNumber());
        }
        else if (c == '-' && (tokens.empty() ||
                 tokens.back().type == TokenType::OPERATOR ||
                 tokens.back().type == TokenType::LEFT_PAREN)) {
            advance();
            if (!isAtEnd() && (std::isdigit(peek()) || peek() == '.')) {
                Token t = readNumber();
                t.number = -t.number;
                t.value = "-" + t.value;
                tokens.push_back(t);
            } else {
                tokens.push_back(Token(TokenType::OPERATOR, "u-"));
            }
        }
        else if (c == '+' || c == '-' || c == '*' ||
                 c == '/' || c == '^' || c == '%') {
            tokens.push_back(Token(TokenType::OPERATOR, std::string(1, c)));
            advance();
        }
        else if (c == '(') {
            tokens.push_back(Token(TokenType::LEFT_PAREN, "("));
            advance();
        }
        else if (c == ')') {
            tokens.push_back(Token(TokenType::RIGHT_PAREN, ")"));
            advance();
        }
        else if (c == ',') {
            tokens.push_back(Token(TokenType::COMMA, ","));
            advance();
        }
        else if (std::isalpha(c) || c == '_') {
            tokens.push_back(readWord());
        }
        else if (c == '=') {
            tokens.push_back(Token(TokenType::OPERATOR, "="));
            advance();
        }
        else {
            throw std::runtime_error(std::string("unexpected character: '") + c + "'");
        }
    }

    tokens.push_back(Token(TokenType::END, ""));
    return tokens;
}

void Tokenizer::skipWhitespace() {
    while (!isAtEnd() && std::isspace(input[pos]))
        ++pos;
}

Token Tokenizer::readNumber() {
    size_t start = pos;
    bool hasDot = false;

    while (!isAtEnd() && (std::isdigit(peek()) || (peek() == '.' && !hasDot))) {
        if (peek() == '.') hasDot = true;
        advance();
    }

    std::string raw = input.substr(start, pos - start);
    return Token(TokenType::NUMBER, raw, std::stod(raw));
}

Token Tokenizer::readWord() {
    size_t start = pos;
    while (!isAtEnd() && (std::isalnum(peek()) || peek() == '_'))
        advance();

    std::string word = input.substr(start, pos - start);

    if (word == "pi") return Token(TokenType::NUMBER, "pi", 3.14159265358979);
    if (word == "e")  return Token(TokenType::NUMBER, "e",  2.71828182845905);
    if (isFunction(word)) return Token(TokenType::FUNCTION, word);

    return Token(TokenType::VARIABLE, word);
}

char Tokenizer::peek() const { return input[pos]; }
char Tokenizer::advance() { return input[pos++]; }
bool Tokenizer::isAtEnd() const { return pos >= input.size(); }