Neha multi model

README.md

@@ -1,4 +1,5 @@
-# EdgeLake Project
+# Federated Learning -- AnyLog EdgeLake
+
 
 <!-- To Do
 
@@ -16,6 +17,8 @@
 
 ## Prerequisites
 <!-- we need specifiy more stuff here -->
+
+- pip install -r requirements.txt
 ### Software Requirements
 - Python 3.7+ installed.
 
@@ -30,6 +33,7 @@ git clone https://github.com/EdgeLake/EdgeLake
 - **master.env**: Ensure this file is available for configuration.                   
 - **start-script.al**: This script is necessary for proper configuration.  
 - **Example CURL commands file**: Keep this handy for testing and reference.
+- **/blockchain/.env**: Used to specify system variables, file paths, etc. Update with values corresponding to your system.
 
 ---
 <!-- I don't think we should do py charm -->
@@ -63,7 +67,113 @@ git clone https://github.com/EdgeLake/EdgeLake
 ### Step 5: Testing with CURL Commands
 <!-- Example curl commands here -->
 
----
+
+'''
+CURL REQUEST FOR DEPLOYING CONTRACT-- General Form
+
+curl -X POST http://localhost:[AGGREGATOR_PORT]/init \
+-H "Content-Type: application/json" \
+-d '{
+  "nodeUrls": [
+    "http://localhost:[NODE_0_PORT]",
+    "http://localhost:[NODE_1_PORT]"
+  ],
+  "model_path": "[FILE_PATH_TO_PYTORCH_MODEL_SOURCE_CODE]",
+  "model_init_params": [OPTIONAL, IF YOUR PYTORCH MODEL HAS ANY],
+  "model_name": "[MODEL_NAME]",
+  "model_weights_path": "[WHERE YOU WANT MODEL WEIGHTS SAVED]",
+  "data_handler_path": "[FILE_PATH_TO_DATA_HANDLER_SOURCE_CODE]",
+  "data_config": {"data": ["[DATA_FILE_FOR_NODE_0]",
+                           "[DATA_FILE_FOR_NODE_1]]}
+}'
+'''
+
+
+'''
+CURL REQUEST FOR DEPLOYING CONTRACT-- custom dataset and model, 1 node 
+
+curl -X POST http://localhost:8080/init \
+-H "Content-Type: application/json" \
+-d '{
+  "nodeUrls": [
+    "http://localhost:8081"
+  ],
+  "model_path": "C:\\Users\\nehab\\cse115d\\testmodel.py",
+  "model_init_params": { "module__input_dim": 14 },
+  "model_name": "custom_test",
+  "model_weights_path": "C:\\Users\\nehab\\cse115d\\model_weights.pt",
+  "data_handler_path": "C:\\Users\\nehab\\cse115d_anylog_edgelake\\custom_data_handler.py",
+  "data_config": {"data": ["C:\\Users\\nehab\\cse115d_anylog_edgelake\\heart_data\\party_data\\party_0.csv"]}
+
+}'
+'''
+
+'''
+CURL REQUEST FOR DEPLOYING CONTRACT-- custom dataset and model, 2 nodes
+
+curl -X POST http://localhost:8080/init \
+-H "Content-Type: application/json" \
+-d '{
+  "nodeUrls": [
+    "http://localhost:8081",
+    "http://localhost:8082"
+  ],
+  "model_path": "C:\\Users\\nehab\\cse115d\\testmodel.py",
+  "model_init_params": { "module__input_dim": 14 },
+  "model_name": "custom_test",
+  "model_weights_path": "C:\\Users\\nehab\\cse115d\\model_weights.pt",
+  "data_handler_path": "C:\\Users\\nehab\\cse115d_anylog_edgelake\\custom_data_handler.py",
+  "data_config": {"data": ["C:\\Users\\nehab\\cse115d_anylog_edgelake\\heart_data\\party_data\\party_0.csv",
+                           "C:\\Users\\nehab\\cse115d_anylog_edgelake\\heart_data\\party_data\\party_1.csv"]}
+}'
+'''
+
+'''
+CURL REQUEST FOR DEPLOYING CONTRACT-- mnist built in dataset and model, 2 nodes
+
+curl -X POST http://localhost:8080/init \
+-H "Content-Type: application/json" \
+-d '{
+  "nodeUrls": [
+    "http://localhost:8081",
+    "http://localhost:8082"
+  ],
+  "model_path": "C:\\Users\\nehab\\cse115d\\Anylog-Edgelake-CSE115D\\federated-learning-lib-main\\examples\\iter_avg\\model_pytorch.py",
+  "model_name": "mnist_test",
+  "model_weights_path": "C:\\Users\\nehab\\cse115d\\model_weights.pt",
+  "data_handler_path": "C:\\Users\\nehab\\cse115d\\Anylog-Edgelake-CSE115D\\venv38\\Lib\\site-packages\\ibmfl\\util\\data_handlers\\mnist_pytorch_data_handler.py",
+  "data_config": {
+    "npz_file": [
+      "C:\\Users\\nehab\\cse115d\\Anylog-Edgelake-CSE115D\\blockchain\\data\\mnist\\data_party0.npz",
+      "C:\\Users\\nehab\\cse115d\\Anylog-Edgelake-CSE115D\\blockchain\\data\\mnist\\data_party1.npz"
+    ]
+  }
+}'
+'''
+
+'''
+CURL REQUEST FOR DEPLOYING CONTRACT-- mnist dataset and sql model, 2 nodes
+
+curl -X POST http://localhost:8080/init \
+-H "Content-Type: application/json" \
+-d '{
+  "nodeUrls": [
+    "http://localhost:8081",
+    "http://localhost:8082"
+  ],
+  "model_path": "C:\\Users\\nehab\\cse115d\\Anylog-Edgelake-CSE115D\\federated-learning-lib-main\\examples\\iter_avg\\model_pytorch.py",
+  "model_name": "mnist_test",
+  "model_weights_path": "C:\\Users\\nehab\\cse115d\\model_weights.pt",
+  "data_handler_path": "C:\\Users\\nehab\\cse115d\\Anylog-Edgelake-CSE115D\\blockchain\\custom_sql_datahandler.py",
+  "data_config": {
+    "npz_file": [
+      "C:\\Users\\nehab\\cse115d\\Anylog-Edgelake-CSE115D\\blockchain\\data\\mnist\\data_party0.npz",
+      "C:\\Users\\nehab\\cse115d\\Anylog-Edgelake-CSE115D\\blockchain\\data\\mnist\\data_party1.npz"
+    ]
+  }
+}'
+'''
+
 
 ## Automating CURL Commands
 <!-- write a script at the end when all the code is ready -->
@@ -90,6 +200,7 @@ for cmd in curl_commands:
 ### Instructions for Script
 
 
+
 ---
 
 ## Troubleshooting
@@ -102,4 +213,3 @@ for cmd in curl_commands:
   - Verify the EdgeLake service is running.
   - Check network connectivity and server status.
 
----

blockchain/.env

@@ -7,6 +7,7 @@ DATABASE_URL=https://anylog-edgelake-fl-default-rtdb.firebaseio.com
 # Ethereum provider URL
 PROVIDER_URL=https://optimism-sepolia.infura.io/v3/6fce3361490c4187b59947005a07c3e7
 
+
 # Ethereum private key
 PRIVATE_KEY=f155acda1fc73fa6f50456545e3487b78fd517411708ffa1f67358c1d3d54977
 
@@ -27,3 +28,4 @@ EXTERNAL_IP=10.0.0.171
 
 # LOCAL PSQL DB NAME
 PSQL_DB_NAME=mnist_fl
+

blockchain/aggregator.py

@@ -13,7 +13,7 @@
 from ibmfl.util.data_handlers.mnist_pytorch_data_handler import MnistPytorchDataHandler
 
 
-CONTRACT_ADDRESS = "0x4ae311B85B017bf7EAa7a96D3109f58795F5F4BF"
+import time
 
 load_dotenv()
 
@@ -51,21 +51,41 @@ def start_round(self, initParamsLink, roundNumber):
             external_ip = os.getenv("EXTERNAL_IP")
             url = f'http://{external_ip}:32049'
 
+            # in khaled's: for node_num in range(1, int(minParams) + 1)
             headers = {
                 'User-Agent': 'AnyLog/1.23',
                 'Content-Type': 'text/plain',
                 'command': 'blockchain insert where policy = !my_policy and local = true and blockchain = optimism'
             }
 
             # Format data exactly like the example curl command but with your values
-            # NOTE: ask why are we adding the node num from agg
-            data = f'''<my_policy = {{"r{roundNumber}" : {{
+            # denote aggregator's params with a 
+            data = f'''<my_policy = {{"a{roundNumber}" : {{
                                         "initParams": "{initParamsLink}"
                               }} }}>'''
 
-            print(f"Training initialized with {roundNumber} rounds")
-
+            # retries = 0;
+            # max_retries = 5;
+            # while retries < max_retries:
+            #     response = requests.post(url, headers=headers, data=data)
+            #     if response.status_code == 200:
+            #         print(f"Aggregator has submitted parameters for round {roundNumber} to the blockchain.")
+            #         return {
+            #             'status': 'success',
+            #             'message': 'Aggregator model parameters added successfully'
+            #         }
+            #     else:
+            #         print(f"Failed to add aggregator params to blockchain. Response: {response}. Retrying ({retries + 1}/{max_retries})...")
+            #         retries += 1;
+            #         time.sleep(15);
+
+            # return {
+            #     'status': 'error',
+            #     'message': 'aggregator was unable to add to blockchain'
+            # }
+
             response = requests.post(url, headers=headers, data=data)
+            print(response.status_code)
             if response.status_code == 200:
                 return {
                     'status': 'success',
@@ -145,6 +165,7 @@ def decode_params(self, encoded_model_update):
         model_weights = pickle.loads(serialized_data)
         return model_weights
 
-    def inference(self, data):
-        results = self.fusion_model.fl_model.predict(data)
+
+    def inference(self, model, data):
+        results = model.evaluate(data)
         return results