main.cpp

#include "./src/tokenizer/tokenizer.hpp"
#include<iostream>
#include<string>
#include<chrono>
#include<windows.h>
#include<psapi.h>

SIZE_T getMemoryUsage() {
    PROCESS_MEMORY_COUNTERS pmc;
    GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc));
    return pmc.WorkingSetSize;  // Return the memory in bytes
}

int main() {

    tokenizer t1;
    tokenizer t2;

    // corpus size(content) : 14663 characters(including spaces)
    // assuming the corpus is already preprocessed

    std::string content = "hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk hello world i am pranav and i slay low level programming fhlkdjf jdlfjadjf kfjalwkdjfoew foiewhf hfioewhnadwll ehfioahnkadwnl fheawoihfsadfnlawd fhoiewhfoaeijf foewiahfawhdfoa fhwjojl fhioewahf afldfhoeiaw fhaewiohf fhoaewfh ndja fheoihf heoiawhf hfow ifhoa fihfoiahf ahfn aohfoe fheoi ahaf aofao aoefo ao fahof aoihof oaf fiojaoij foiaw fioa ofoiehaoiehj fioa of eoahfoa fio afoihaof oaefoai foiaew fioa ifhioea fhoie fjahdf fladsjfkl adsfdfk";

    // std::string content = "hello world i am pranav and i slay low level programming";


    // // calculating time taken for sequential tokenization
    auto start = std::chrono::high_resolution_clock::now();
    _type_token tokens1 = t1.tokenize(content);
    auto end = std::chrono::high_resolution_clock::now();
    auto sequential_time = std::chrono::duration<float>(end - start).count();
    std::cout<<"\n\nSequential Tokenization : "<<sequential_time * 1000<<" ms"<<std::endl;

    // calculating time taken for parallel tokenization
    start = std::chrono::high_resolution_clock::now();
    _type_token tokens2 = t2.tokenize_multithreaded(content, 2);
    end = std::chrono::high_resolution_clock::now();
    auto parallel_time = std::chrono::duration<float>(end - start).count();
    std::cout<<"Multithreaded Tokenization : "<<parallel_time * 1000<<" ms"<<std::endl;

    float speed_up = sequential_time / parallel_time;
    std::cout<<"Speed Up : "<<speed_up<<std::endl;
    std::cout<<"Corpus Size(chars) : "<<content.length()<<std::endl;
    std::cout<<"Total Tokens : "<<tokens2.size()<<std::endl;
    std::cout<<"Num Threads : 2"<<std::endl;
    const float mem_usage = (getMemoryUsage() / 1000000.f);    // converting bytes to mega-bytes
    std::cout << "Memory Usage: " << mem_usage << " MB\n\n" << std::endl;


    return 0;
}