Issue#7 alexandru alexandru

backend/pytorch.py

@@ -0,0 +1,163 @@
+import torch
+import torch.nn as nn
+import torch.optim as optim
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+
+
+# Necesare suprascrierea functiei __len__ si supraincarcarea operatorului []
+class GradesDataset(torch.utils.data.Dataset):
+    def __init__(self, dataFrame):
+        self._df = dataFrame
+
+    def __len__(self):
+        return len(self._df)
+
+    def __getitem__(self, idx):  # []
+        item = self._df.iloc[idx]
+        features = []
+        for feature in self._df.columns[2:-1]:
+            features.append(item[feature])
+
+        return {'features': torch.tensor(features, dtype=torch.float32),
+                'labels': torch.tensor([item['grade']], dtype=torch.float32)}
+
+
+# Necesare a fi suprascrise metodele __init__ si forward
+class RegressionModel(nn.Module):
+    def __init__(self, in_feat=2, out_feat=1):
+        super().__init__()
+        self.fc = nn.Linear(in_feat, out_feat)
+
+    def forward(self, x):
+        out = self.fc(x)
+        return out
+
+def getTestSet():
+    # Citim datele din fisierele .csv
+    featuresFrame = pd.read_csv('../data/features.csv')
+    gradesFrame = pd.read_csv('../data/grades.csv')
+
+    # Concatenam ambele tabele
+    dataFrame = pd.merge(featuresFrame, gradesFrame, on="label")
+    dataFrame = dataFrame[dataFrame['grade'].notna()]
+
+
+    # Stergem coloana Unnamed: 22
+    dataFrame.drop(['Unnamed: 22'], axis=1, inplace=True)
+
+    # Impartim dataFrame-ul in doua frame-uri
+    # unul pentru antrenare si unul pentru test
+    train_frame, test_frame = train_test_split(dataFrame, test_size=0.2)
+    return GradesDataset(test_frame)
+
+def getInputNumber():
+    # Citim datele din fisierele .csv
+    featuresFrame = pd.read_csv('../data/features.csv')
+    gradesFrame = pd.read_csv('../data/grades.csv')
+
+    # Concatenam ambele tabele
+    dataFrame = pd.merge(featuresFrame, gradesFrame, on="label")
+    dataFrame = dataFrame[dataFrame['grade'].notna()]
+
+    # Stergem coloana Unnamed: 22
+    dataFrame.drop(['Unnamed: 22'], axis=1, inplace=True)
+
+    # Numarul de feature-uri din set-ul dataFrame
+    input_number = len(dataFrame.columns[2:-1])
+    return input_number
+
+def our_train():
+    # Citim datele din fisierele .csv
+    featuresFrame = pd.read_csv('../data/features.csv')
+    gradesFrame = pd.read_csv('../data/grades.csv')
+
+    # Concatenam ambele tabele
+    dataFrame = pd.merge(featuresFrame, gradesFrame, on="label")
+    dataFrame = dataFrame[dataFrame['grade'].notna()]
+
+
+    # Stergem coloana Unnamed: 22
+    dataFrame.drop(['Unnamed: 22'], axis=1, inplace=True)
+
+    # Impartim dataFrame-ul in doua frame-uri
+    # unul pentru antrenare si unul pentru test
+    train_frame, test_frame = train_test_split(dataFrame, test_size=0.2)
+
+    # Creeam set-urile de date
+    train_set = GradesDataset(train_frame)
+    test_set = GradesDataset(test_frame)
+
+    # Numarul de feature-uri din set-ul dataFrame
+    input_number = len(dataFrame.columns[2:-1])
+    model = RegressionModel(input_number, 1)
+
+    LEARNING_RATE = 1e-10  # Rata de invatare
+    NR_EPOCHS = 6  # Numarul de epoci
+    BATCH_SIZE = 32  # Numarul de samples dintr-un batch
+
+    criterion = nn.MSELoss()  # functia cost (loss)
+    # algoritmul de optimizare (Stochastic Gradient Descent)
+    optimizer = optim.SGD(model.parameters(), lr=LEARNING_RATE)
+
+
+    # Pregatim o modalitate de loggare a informatiilor din timpul antrenarii
+    log_info = []
+
+    # Pregatim DataLoader-ul pentru antrenare
+    train_loader = torch.utils.data.DataLoader(
+        train_set, batch_size=BATCH_SIZE, shuffle=True)
+
+    # Trecem modelul in modul train
+    model.train()
+
+
+    ########### Training Loop #############
+
+    # pentru fiecare epoca (1 epoca = o iteratie peste intregul set de date)
+    for epoch in range(NR_EPOCHS):
+        print('Running epoch {}'.format(epoch))
+
+        epoch_losses = []
+
+        # pentru fiecare batch de BATCH_SIZE exemple din setul de date
+        for i, batch in enumerate(train_loader):
+
+            inputs, labels = batch['features'], batch['labels']
+
+            # anulam gradientii deja acumulati la nivelul retelei neuronale
+            optimizer.zero_grad()
+
+            # FORWARD PASS: trecem inputurile prin retea
+            outputs = model(inputs)
+
+            # Calculam LOSSul dintre etichetele prezise si cele reale
+            loss = criterion(outputs, labels)
+
+            # BACKPRPAGATION: calculam gradientii propagand LOSSul in retea
+            loss.backward()
+
+            # Utilizam optimizorul pentru a modifica parametrii retelei in functie de gradientii acumulati
+            optimizer.step()
+
+            # Salvam informatii despre antrenare (in cazul nostru, salvam valoarea LOSS)
+            epoch_losses.append(loss.item())
+        log_info.append((epoch, np.mean(epoch_losses)))
+
+    # Graficul LOSS-ului pe parcursul antrenarii
+    # X = [x for x, loss in log_info]
+    # Y = [loss for x, loss in log_info]
+    # plt.plot(X, Y)
+    # plt.xlabel("Epoch")
+    # plt.ylabel("LOSS")
+    # plt.show()
+
+    # print(model)
+    # print(model.fc.weight, model.fc.bias)
+
+
+    torch.save(model.state_dict(), "../data/model.pt")
+

backend/server.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 from flask import Flask, request, jsonify
 from flask_cors import CORS
 from Crypto.Hash import MD5
@@ -11,7 +10,8 @@
 import warnings
 import feature_extraction
 from preprocessor import Preprocessor
-from train import Train, Predictor
+from train import Train, Predictor, PredictorTorch
+import pytorch
 
 DOWNLOAD_DIRECTORY = "uploads"
 EXTRACTION_DIRECTORY = "../data/raw/train"
@@ -57,6 +57,7 @@ def predict_route():
         zip_file.extractall(extraction_full_path)
 
     # Get features
+    print(extraction_full_path)
     features = feature_extraction.retrain_data_one(extraction_full_path + "/")
     features = preprocessor.transform_entry(features)
 
@@ -75,6 +76,63 @@ def predict_route():
     return jsonify(result)
 
 
+# Prediction route for multiple files
+@app.route("/predict_multiple", methods=["POST"])
+def predict_multiple_route():
+
+    global last_student_scanned, preprocessor, predictor
+
+    # Get arguments
+    uploaded_file = request.files["file"]
+
+    # Generate a filename and save the file locally
+    unique_filename = uploaded_file.filename + str(time.time())
+    unique_filename = MD5.new(unique_filename.encode("utf-8")).hexdigest()
+    last_student_scanned = unique_filename
+    full_path = os.path.join(DOWNLOAD_DIRECTORY, unique_filename + ".zip")
+    uploaded_file.save(full_path)
+
+    # Extract the uploaded archive
+    extraction_full_path = os.path.join(EXTRACTION_DIRECTORY, unique_filename)
+    os.makedirs(extraction_full_path)
+
+    grade_list = []
+
+    with zipfile.ZipFile(full_path, "r") as zip_file:
+        zip_file.extractall(extraction_full_path)
+
+        dirs = list(set([os.path.dirname(x) for x in zip_file.namelist()]))
+        topdirs = [os.path.split(x)[0] for x in dirs]
+
+        mylist = []
+        for elem in topdirs:
+            if elem.count('/') == 2:
+                mylist.append(elem)
+
+        mylist.sort()
+        for x in mylist:
+            # Get features
+            features = feature_extraction.retrain_data_one(extraction_full_path + "/" + x + "/")
+            features = preprocessor.transform_entry(features)
+
+            # Predict the grade
+            grade = predictor.predict([features])[0]
+            grade = round(grade, 2)
+
+            # Dump the grade into the specific CSV file
+            grades_df = pandas.read_csv(GRADES_CSV_FILENAME)
+            grades_df.loc[len(grades_df.index)] = [last_student_scanned, grade]
+            grades_df = grades_df[["label", "grade"]]
+            grades_df.to_csv(GRADES_CSV_FILENAME, index=False)
+
+            print(grade)
+            grade_list.append(grade)
+
+    # Return a result
+    result = {"predicted_grade": grade_list}
+    return jsonify(result)
+
+
 # Grade adjusting route
 @app.route("/adjust_grade", methods=["GET"])
 def grade_adjustment_route():
@@ -100,21 +158,33 @@ def grade_adjustment_route():
 @app.route("/retrain_model", methods=["GET"])
 def model_retraining_route():
 
+    # Get arguments
+    model = request.args.get("model", type=int)
+
     # Create a thread that retrain the model
-    Thread(target=retrain_model).start()
+    Thread(target=retrain_model(model)).start()
 
     # Return a result
     result = {"status": "ok"}
     return jsonify(result)
 
 
 # Function for retraining the machine learning model
-def retrain_model():
+def retrain_model(model):
     global predictor, preprocessor
 
-    Train(check=True).train()
-    predictor = Predictor()
-    preprocessor = Preprocessor()
+    # Train(check=True).train()
+    if (model == 1):
+        Train(check=True).train()
+    else:
+        pytorch.our_train()
+
+    if (model == 1):
+        predictor = Predictor()
+    else:
+        predictor = PredictorTorch()
+
+    preprocessor = Preprocessor() 
 
     print("[+] Successfully retrained the model")
 

backend/train.py

@@ -6,11 +6,15 @@
 from sklearn.linear_model import LinearRegression, SGDRegressor, Ridge
 from sklearn.metrics import mean_squared_error
 from sklearn.svm import SVR
+import torch
 import math
 import os
 import joblib
 
+import pytorch
+
 WEIGHTS_FILE = "../data/weights.apt"
+OUR_MODEL_FILE = "../data/model.pt"
 
 # Training class
 # Allows for two methods - RandomForest and LinearRegression
@@ -150,19 +154,55 @@ def _load_true_notes(self):
 
         return array
 
-
 class Predictor:
     def __init__(self, model_name=WEIGHTS_FILE):
 
         self.model = joblib.load(model_name)
+        print(self.model)
 
     # returns <class 'numpy.ndarray'>
     def predict(self, features):
 
         prediction = self.model.predict(features)
+        print(prediction)
 
         return prediction
 
+class PredictorTorch:
+    def __init__(self, model_name=OUR_MODEL_FILE):
+
+        self.model = pytorch.RegressionModel(pytorch.getInputNumber(),1)
+        self.model.load_state_dict(torch.load(model_name))
+        self.model.eval()
+        print(self.model)
+
+    # returns <class 'numpy.ndarray'>
+    def predict(self, features):
+        # Pregatim DataLoader-ul pentru validare
+        test_loader = torch.utils.data.DataLoader(pytorch.getTestSet(), batch_size=32, shuffle=False)
+
+        # Pregatim o modalitate de stocare a datelor pentru evaluare
+        eval_outputs = []
+        true_labels = []
+        x = []
+
+        # ########### Evaluation Loop #############
+        print(test_loader)
+        with torch.no_grad():
+            for batch in test_loader:
+                inputs, labels = batch['features'], batch['labels']
+                # calculate outputs by running images through the network
+                outputs = self.model(inputs)
+                eval_outputs += outputs.squeeze(dim=1).tolist()
+                true_labels += labels.squeeze(dim=1).tolist()
+                x += inputs.squeeze(dim=1).tolist()
+
+        sum = 0
+        for i in true_labels:
+            sum += i
+
+        return [sum/len(true_labels)]
+
 
 #How to run