functions_helpers_eval.py

import re
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

def handle_negation(text):
    # Expand contractions
    text = text.lower()
    text = re.sub(r"\b(can't)\b", "can not", text)
    text = re.sub(r"\b(don't)\b", "do not", text)
    text = re.sub(r"\b(doesn't)\b", "does not", text)
    text = re.sub(r"\b(won't)\b", "will not", text)
    text = re.sub(r"\b(isn't)\b", "is not", text)
    text = re.sub(r"\b(aren't)\b", "are not", text)
    text = re.sub(r"\b(hasn't)\b", "has not", text)
    text = re.sub(r"\b(haven't)\b", "have not", text)
    text = re.sub(r"\b(shouldn't)\b", "should not", text)

    # Actually handling negations
    text = re.sub(r"not\s+(\w+)", r"not_\1", text)
    return text

def remove_special_characters(text):
    return re.sub(r"[^a-z\s_]", '', text)


def tokenize_text(text):
    """Tokenizes text into words."""
    return word_tokenize(text)


def remove_stopwords(tokens):
    """Removes stopwords from a tokenized list."""
    stop_words = set(stopwords.words("english"))
    return [word for word in tokens if word.lower() not in stop_words]


def lemmatize_tokens(tokens):
    """Lemmatizes each token (reduces to base form)."""
    lemmatizer = WordNetLemmatizer()
    return [lemmatizer.lemmatize(token) for token in tokens]

def preprocess_LLM_words(issues):
    processed_issues = []

    for issue in issues:
        words = []
        for phrase in issue["words"]:
            phrase = handle_negation(phrase)  # Step 1: Handle negations and lower letters
            phrase = remove_special_characters(phrase)  # Step 2: Remove special characters
            tokens = tokenize_text(phrase)  # Step 3: Tokenize
            tokens = remove_stopwords(tokens)  # Step 4: Remove stopwords
            lemmatized_tokens = lemmatize_tokens(tokens)  # Step 5: Lemmatize

            words.extend(lemmatized_tokens)  # Flatten list

        unique_words = list(dict.fromkeys(words))
        processed_issues.append({"name": issue["name"], "words": unique_words})
    return processed_issues