Skip to content

Commit 7399604

Browse files
Muntasib-creatorMuntasib-creator
committed
nlp action added
1 parent cf3417c commit 7399604

5 files changed

Lines changed: 231 additions & 0 deletions

File tree

‎Framework/Built_In_Automation/AI/BuiltInFunctions.py‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
import sys, os, inspect
2+
from pathlib import Path
3+
import base64
4+
from io import BytesIO
5+
6+
sys.path.append("..")
7+
from selenium import webdriver
8+
9+
from Framework.Utilities import CommonUtil, ConfigModule
10+
from Framework.Built_In_Automation.Shared_Resources import (
11+
BuiltInFunctionSharedResources as Shared_Resources,
12+
)
13+
from Framework.Utilities.decorators import logger
14+
15+
try:
16+
import pyautogui
17+
except:
18+
pyautogui = None
19+
20+
try:
21+
from PIL import Image
22+
except:
23+
Image = None
24+
25+
MODULE_NAME = inspect.getmodulename(__file__)
26+
27+
StepDataType = list[list[str, str, str]]
28+
ReturnType = "passed" | "zeuz_failed"
29+
30+
temp_config = os.path.join(
31+
os.path.join(
32+
os.path.abspath(__file__).split("Framework")[0],
33+
os.path.join(
34+
"AutomationLog", ConfigModule.get_config_value("Advanced Options", "_file")
35+
),
36+
)
37+
)
38+
39+
@logger
40+
def natural_language_action(step_data: StepDataType) -> ReturnType:
41+
sModuleInfo = inspect.currentframe().f_code.co_name + " : " + MODULE_NAME
42+
43+
44+
images = []
45+
texts = []
46+
action_type = ""
47+
var_name = ""
48+
instruction = ""
49+
50+
try:
51+
for left, mid, right in step_data:
52+
left_lower = left.lower().strip()
53+
mid_lower = mid.lower().strip()
54+
right_stripped = right.strip()
55+
56+
if left_lower == "image":
57+
if right_stripped == "selenium_screenshot":
58+
if Shared_Resources.Test_Shared_Variables("selenium_driver"):
59+
selenium_driver = Shared_Resources.Get_Shared_Variables("selenium_driver")
60+
else:
61+
selenium_driver = None
62+
if selenium_driver:
63+
screenshot = selenium_driver.get_screenshot_as_png()
64+
images.append({
65+
"type": "selenium_screenshot",
66+
"data": base64.b64encode(screenshot).decode('utf-8')
67+
})
68+
CommonUtil.ExecLog(sModuleInfo, "Captured Selenium screenshot", 1)
69+
else:
70+
CommonUtil.ExecLog(sModuleInfo, "Selenium driver not available", 2)
71+
72+
elif right_stripped == "desktop_screenshot":
73+
if pyautogui and Image:
74+
try:
75+
screenshot = pyautogui.screenshot()
76+
buffered = BytesIO()
77+
screenshot.save(buffered, format="PNG")
78+
img_data = base64.b64encode(buffered.getvalue()).decode('utf-8')
79+
images.append({
80+
"type": "pyautogui_screenshot",
81+
"data": img_data
82+
})
83+
CommonUtil.ExecLog(sModuleInfo, "Captured PyAutoGUI screenshot", 1)
84+
except Exception as e:
85+
CommonUtil.ExecLog(
86+
sModuleInfo,
87+
f"Failed to capture desktop screenshot (headless environment?): {str(e)}",
88+
2
89+
)
90+
else:
91+
CommonUtil.ExecLog(sModuleInfo, "PyAutoGUI not available", 2)
92+
else:
93+
image_path = CommonUtil.path_parser(right_stripped)
94+
if os.path.exists(image_path):
95+
with open(image_path, "rb") as image_file:
96+
image_data = image_file.read()
97+
images.append({
98+
"type": "image",
99+
"data": base64.b64encode(image_data).decode('utf-8')
100+
})
101+
else:
102+
CommonUtil.ExecLog(sModuleInfo, f"Image file not found: {image_path}", 2)
103+
images.append({
104+
"type": "image",
105+
"data": right_stripped
106+
})
107+
elif left_lower == "text":
108+
if right_stripped == "selenium_dom":
109+
if selenium_driver:
110+
dom = Shared_Resources.get_cleaned_selenium_dom(selenium_driver)
111+
if dom:
112+
texts.append({
113+
"type": "dom",
114+
"data": dom
115+
})
116+
else:
117+
CommonUtil.ExecLog(sModuleInfo, "Failed to capture cleaned DOM", 2)
118+
else:
119+
CommonUtil.ExecLog(sModuleInfo, "Selenium driver not available for DOM", 2)
120+
121+
else:
122+
texts.append({
123+
"type": "text",
124+
"data": right_stripped
125+
})
126+
127+
elif left_lower == "action":
128+
action_type = right_stripped
129+
130+
elif left_lower == "natural language action":
131+
var_name = right_stripped
132+
133+
result_data = {
134+
"images": images,
135+
"texts": texts,
136+
"action": action_type,
137+
"instruction": instruction
138+
}
139+
140+
payload_dir = Path(__file__).parent / "payload"
141+
payload_dir.mkdir(exist_ok=True)
142+
143+
for img in images:
144+
if img["type"] == "selenium_screenshot":
145+
img_path = payload_dir / "selenium_screenshot.png"
146+
img_bytes = base64.b64decode(img["data"])
147+
with open(img_path, "wb") as f:
148+
f.write(img_bytes)
149+
CommonUtil.ExecLog(sModuleInfo, f"Saved selenium screenshot to {img_path}", 1)
150+
151+
for text in texts:
152+
if text["type"] == "dom":
153+
dom_path = payload_dir / "selenium_dom.txt"
154+
with open(dom_path, "w", encoding="utf-8") as f:
155+
f.write(text["data"])
156+
CommonUtil.ExecLog(sModuleInfo, f"Saved selenium DOM to {dom_path}", 1)
157+
158+
text_counter = 1
159+
for text in texts:
160+
if text["type"] == "text":
161+
text_path = payload_dir / f"text{text_counter}.txt"
162+
with open(text_path, "w", encoding="utf-8") as f:
163+
f.write(text["data"])
164+
CommonUtil.ExecLog(sModuleInfo, f"Saved text to {text_path}", 1)
165+
text_counter += 1
166+
167+
image_counter = 1
168+
for img in images:
169+
if img["type"] in ["image", "pyautogui_screenshot"]:
170+
img_path = payload_dir / f"image{image_counter}.png"
171+
img_bytes = base64.b64decode(img["data"])
172+
with open(img_path, "wb") as f:
173+
f.write(img_bytes)
174+
CommonUtil.ExecLog(sModuleInfo, f"Saved image to {img_path}", 1)
175+
image_counter += 1
176+
177+
CommonUtil.ExecLog(
178+
sModuleInfo,
179+
f"Natural language action executed with {len(images)} images, {len(texts)} texts",
180+
1
181+
)
182+
183+
return Shared_Resources.Set_Shared_Variables(var_name, result_data)
184+
185+
except Exception:
186+
return CommonUtil.Exception_Handler(sys.exc_info())
187+
188+
189+
@logger
190+
def switch_iframe(step_data: StepDataType) -> ReturnType:
191+
sModuleInfo = inspect.currentframe().f_code.co_name + " : " + MODULE_NAME
192+
193+
try:
194+
selenium_driver = Shared_Resources.Get_Shared_Variables("selenium_driver")
195+
except:
196+
selenium_driver = None
197+
198+
try:
199+
for left, mid, right in step_data:
200+
left = left.lower().strip()
201+
if "action" in mid.lower() and left == "switch iframe":
202+
pass
203+
elif left == "index" and "default" in right.lower():
204+
if selenium_driver:
205+
selenium_driver.switch_to.default_content()
206+
CommonUtil.ExecLog(
207+
sModuleInfo, "Exited all iframes and switched to default content", 1
208+
)
209+
return "passed"
210+
except Exception:
211+
return CommonUtil.Exception_Handler(sys.exc_info())
212+

‎Framework/Built_In_Automation/AI/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import sys
2+
3+
sys.path.append("..")

‎Framework/Built_In_Automation/Sequential_Actions/action_declarations/ai.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
declarations = (
2+
{ "name": "natural language action", "function": "natural_language_action", "screenshot": "web" },
3+
) # yapf: disable
4+
5+
module_name = "ai"
6+
7+
for dec in declarations:
8+
dec["module"] = module_name

‎Framework/Built_In_Automation/Sequential_Actions/action_declarations/info.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import regex as re
22
from . import (
3+
ai,
34
appium,
45
common,
56
desktop,
@@ -15,6 +16,7 @@
1516
)
1617

1718
modules = (
19+
ai,
1820
appium,
1921
common,
2022
desktop,
@@ -43,6 +45,7 @@
4345
"action",
4446
"optional action",
4547
"loop action",
48+
"ai action",
4649
"element parameter",
4750
"child parameter",
4851
"sibling parameter",

‎Framework/Built_In_Automation/Sequential_Actions/sequential_actions.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,11 @@ def load_sa_modules(
157157
from Framework.Built_In_Automation.Security import (
158158
BuiltInFunctions as security
159159
)
160+
elif module == "ai":
161+
global ai
162+
from Framework.Built_In_Automation.AI import (
163+
BuiltInFunctions as ai
164+
)
160165
else:
161166
CommonUtil.ExecLog(
162167
sModuleInfo, "Invalid sequential actions module: %s" % module, 3

0 commit comments

Comments
 (0)