{{ transcript }}
""" def generate_frames(): while True: ## read the camera frame success,frame=camera.read() if not success: break else: ret,buffer=cv2.imencode('.jpg',frame) frame=buffer.tobytes() yield(b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n') @app.route("/") def test(): return jsonify("Hola Mundo") @app.route("/generated", methods=["GET"]) def generated(): data = request.args response = {"received_data": data.to_dict()} print(response) return jsonify(response) @app.route("/descriptions") def upload_file(): if 'file' not in request.files: return 'No file part' file = request.files['file'] if file.filename == '': return 'No selected file' if file: file.save('uploads/' + file.filename) return 'File uploaded successfully' @app.route('/stream') def stream(): return render_template("./index.html") @app.route('/video') def video(): return Response(generate_frames(),mimetype='multipart/x-mixed-replace; boundary=frame') @app.route('/grab-frame') def grab_frame(): url = serber + 'video' # Open the video stream using OpenCV cap = cv2.VideoCapture(url) # Check if the video capture was successful if not cap.isOpened(): return "Unable to open video stream", 500 # Read a single frame from the video stream ret, frame = cap.read() # Release the video capture object cap.release() if not ret or frame is None: return "No frame available", 404 # Encode the frame as JPEG ret, jpeg = cv2.imencode('.jpg', frame) if not ret: return "Failed to encode frame", 500 # Convert the JPEG image to bytes image_bytes = jpeg.tobytes() # Create an in-memory bytes buffer buffer = io.BytesIO(image_bytes) # Send the image as a downloadable file buffer.seek(0) return send_file(buffer, mimetype='image/jpeg', as_attachment=True, download_name='frame.jpg') @app.route('/compare-input-vs-instr') def compare(): global step_tracker instruction = get_instruction(step_tracker) image_summary = generate_full_summary() audio_summary = latest_transcript push_latest_transcript_to_sql() resp = checkai_instr_vs_summaries(instruction, image_summary, audio_summary) chat_log = update_sql_base(audio_summary, resp) if resp.split('. ')[-1] == "Proceed.": step_tracker += 1 try: # Send a POST request to the agent's endpoint agent_response = requests.post("http://localhost:8001/rest", json=chat_log) # Check if the agent responded successfully if agent_response.status_code == 200: print("Successfully forwarded data to the agent.") return jsonify({"data": chat_log, "status": "success"}), 200 else: print(f"Agent response error: {agent_response.status_code}") return {"status": "error", "message": "Agent processing failed"}, 500 except Exception as e: print(f"Error forwarding data to the agent: {e}") return {"status": "error", "message": "Failed to forward to agent"}, 500 def push_latest_transcript_to_sql(): try: with connection.cursor() as cursor: # SQL query #query = f"INSERT INTO Step_{step_tracker} (ID,SPEAKER,MESSAGE) VALUES (%s, %s, %s);", (idx + 1, "User", user) # Execute the query cursor.execute(f"INSERT INTO Transcript (STEP,TEXT) VALUES (%s, %s);", (step_tracker, latest_transcript)) connection.commit() return f"Step: {step_tracker}\nTranscript: {latest_transcript}" except pymysql.MySQLError as e: print(f"Error: {e}") return f"Error: {e}" def get_last_index(step_tracker): with connection.cursor() as cursor: search_query = f"SELECT MAX(id) AS highest_index FROM Step_{step_tracker}" cursor.execute(search_query) results = cursor.fetchall() return results[0][0] def update_sql_base(user, chat): global step_tracker idx = get_last_index(step_tracker) try: with connection.cursor() as cursor: # SQL query #query = f"INSERT INTO Step_{step_tracker} (ID,SPEAKER,MESSAGE) VALUES (%s, %s, %s);", (idx + 1, "User", user) # Execute the query cursor.execute(f"INSERT INTO Step_{step_tracker} (ID,SPEAKER,MESSAGE) VALUES (%s, %s, %s);", (idx + 1, "User", user)) cursor.execute(f"INSERT INTO Step_{step_tracker} (ID,SPEAKER,MESSAGE) VALUES (%s, %s, %s);", (idx + 2, "Ju [truncated — 1412 more characters] ``` ### singlestore.py ```python import pymysql import openai # Connect to the SingleStore database connection = pymysql.connect( host='svc-f90325a9-8b27-495d-a436-7cb5c7764c62-dml.aws-oregon-3.svc.singlestore.com', user='admin', password='Flj3M3k6N1of0CXLuR73YiPRMkf9JiTj', database='instructions', port=3306 ) # Function to create a chat log table for a specific step with an id column def create_chat_log_table(table_name): with connection.cursor() as cursor: sql = f""" CREATE TABLE IF NOT EXISTS `{table_name}` ( id INT, Speaker VARCHAR(50), Message TEXT ); """ cursor.execute(sql) connection.commit() print(f"Table '{table_name}' created successfully!") openai.api_key = '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' # Function to get the simplified translation of a medical procedure step def get_simplified_instruction(step_instruction): prompt = f"Please simplify the following medical procedure step into a more convenient and understandable instruction for the user: {step_instruction}" response = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "system", "content": "You are a helpful assistant that simplifies medical instructions."}, {"role": "user", "content": prompt} ], max_tokens=200, temperature=0.7 ) # Extract and return the simplified instruction from the response simplified_instruction = response['choices'][0]['message']['content'].strip() print(f"Simplified instruction generated: {simplified_instruction}") return simplified_instruction # Function to insert a message into a chat log table def insert_chat_log(table_name, speaker, message, idx): with connection.cursor() as cursor: cursor.execute(f"INSERT INTO `{table_name}` (ID, Speaker, Message) VALUES (%s, %s, %s)", (idx, speaker, message)) connection.commit() steps = [ "Step 1: Gather all essential equipment. Sterile attire: Sterile gown, Mask, Gloves, Drapes; Dressings and tape: Petroleum-based gauze dressings, Regular gauze dressings, Tape; Cleansing solution: 2% chlorhexidine solution; Needles: 25-gauge needles for skin infiltration, 21-gauge needles for deeper tissues; Syringes: 10-mL syringes, 20-mL syringes; Local anesthetic: 1% lidocaine; Clamps: Two hemostat or Kelly clamps; Sutures: Nonabsorbable, strong silk or nylon suture (e.g., 0 or 1-0); Scalpel: Size 11 blade; Chest tube for surgical tube thoracostomy: Sizes range from 16 to 36 French (Fr), depending on intended use, 20 to 24 Fr for pneumothorax or malignant pleural effusion, 28 to 36 Fr for complicated parapneumonic effusions, empyema, bronchopleural fistula, 32 to 36 Fr for hemothorax; Thoracostomy catheter (pigtail catheter) for catheter thoracostomy: ≤14 Fr; Suction device; Water seal drainage apparatus and connecting tubing.", "Step 2: Review additional considerations and relevant anatomy. Expertise required: Procedure should be performed by a physician trained in chest tube insertion; Inpatient procedure: Chest tube placement requires hospital admission; Catheter selection: Use smaller diameter catheters (≤14 Fr) for pneumothoraces or free-flowing effusions, Use larger diameter tubes for purulent effusions and hemothoraces to prevent clogging and kinking; Pain management: Smaller catheters cause less pain and may not require sutures after removal; Relevant anatomy: Neurovascular bundles are located at the lower edge of each rib, Critical point: The tube must be placed over the upper edge of the rib to avoid damaging the neurovascular bundle. Hazard: Misplacement can cause injury to neurovascular structures, leading to bleeding or intercostal neuralgia.", "Step 3: Position the patient with the head of the bed elevated at 30 to 60 degrees. Purpose: Limits diaphragm elevation during expiration, reducing the risk of inadvertent intra-abdominal tube placement.", "Step 4: Position the patient's arm on the affected side. Options: Place the arm over the patient's head or abducted to expose the lateral chest wall, Alternatively, have the patient place their hand behind their head.", "Step 5: Connect the water seal suction apparatus to a suction source. Equipment: Ensure it is sealed with sterile water and connected using appropriate tubing. Hazard: The apparatus must be kept 100 cm (40 inches) below the patient to prevent retrograde flow of air or fluid into the pleural space.", "Step 6: Select and mark the insertion site. For pneumothorax: 4th intercostal space at the mid-axillary or anterior axillary line; For other indications: 5th intercostal space at the mid-axillary or anterior axillary line. Tip: Use a skin marking pen or make an impression with a pen before skin preparation.", "Step 7: Prepare the insertion site with an antiseptic solution. Equipment: Use 2% chlorhexidine solution. Hazard: Ensure thorough cleansing to prevent infection.", "Step 8: Drape the area with sterile drapes to maintain a sterile field.", "Step 9: Administer local anesthesia using 1% lidocaine. Equipment: 25-gauge needle for skin infiltration, 21-gauge needle for deeper tissues. Procedure: Inject into the skin, subcutaneous tissue, rib periosteum (of the rib below the insertion site), and parietal pleura. Use generous amounts around the highly pain-sensitive periosteum and parietal pleura. Hazard: Aspirate before injecting to avoid intravascular injection. Confirmation: Entry into the pleural space is confirmed by aspiration of air (in pneumothorax) or fluid (in effusion).", "Step 10: Estimate the insertion depth for the chest tube or catheter. Procedure: Ensure all side holes of the tube or catheter will be inside the pleural space, Account for subcutaneous fat, especially in obese patients [truncated — 9085 more characters] ``` ### backend/agent1.py ```python from utils.config import setup_logging from agents.assistant import assistant_agent def main(): setup_logging() assistant_agent.run() if __name__ == "__main__": main() ``` ### backend/agent2.py ```python from utils.config import setup_logging # from agents.user import user_agent from agents.userinput import user_agent def main(): setup_logging() user_agent.run() if __name__ == "__main__": main() ``` ### image_descrip_test/drop_tables.py ```python import pymysql # Establish connection to the database connection = pymysql.connect( host='svc-f90325a9-8b27-495d-a436-7cb5c7764c62-dml.aws-oregon-3.svc.singlestore.com', user='admin', password='Flj3M3k6N1of0CXLuR73YiPRMkf9JiTj', database='instructions', port=3306 ) try: # Ensure cursor is created and managed properly with connection.cursor() as cursor: # Query to get all the tables in the database cursor.execute("SHOW TABLES") tables = cursor.fetchall() if not tables: print("No tables found in the database.") else: # Loop through and drop each table for table in tables: table_name = table[0] # Extract table name from tuple drop_query = f"DROP TABLE IF EXISTS {table_name}" cursor.execute(drop_query) print(f"Dropped table: {table_name}") # Commit the changes to the database connection.commit() except pymysql.MySQLError as e: print(f"Error: {e}") finally: # Close the database connection connection.close() print("All tables dropped and connection closed.") ``` ### image_descrip_test/image_description.py ```python import os import openai from PIL import Image import torch from transformers import BlipProcessor, BlipForConditionalGeneration # Set your API key openai.api_key = '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' def get_in_depth_summary(caption): prompt = f"Provide a thorough and detailed analysis, including possible contexts, emotions, and implications, based on the following image description:\n\n'{caption}'" response = openai.ChatCompletion.create( engine='gpt-4o', prompt=prompt, max_tokens = 500, temperature=0.7, n=1, stop=None ) summary = response['choices'][0]['text'].strip() return summary try: # Load and process the image image_path = os.path.join('images', 'pfp.png') image = Image.open(image_path).convert('RGB') # Generate a caption using BLIP processor = BlipProcessor.from_pretrained('Salesforce/blip-image-captioning-base') model = BlipForConditionalGeneration.from_pretrained('Salesforce/blip-image-captioning-base') inputs = processor(image, return_tensors='pt') with torch.no_grad(): out = model.generate(**inputs, max_length = 50) caption = processor.decode(out[0], skip_special_tokens=True) print(f"Generated Caption: {caption}") # Get an in-depth summary from OpenAI summary = get_in_depth_summary(caption) print(f"\nIn-Depth Summary:\n{summary}") except Exception as e: print(f"An error occurred: {e}") ``` ### server/test.py ```python import openai import base64 # Set your OpenAI API key openai.api_key = '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' client = openai.OpenAI( api_key=openai.api_key ) # Define the image file path and the prompt image_path = '/Users/amanp/Desktop/code/Calhacks/server/images/pfp.png' prompt = "Describe in detail everything that is visible in the image provided. Include all objects, people, and background elements. Mention the types of objects, their colors, positions, and any noticeable features or activities. If there are any brands, logos, or text, include those as well. Describe the setting, such as the type of location or event, and any other context that can be inferred. Be thorough and list as many elements as possible, even if they seem small or insignificant." img_type = "image/png" with open(image_path, "rb") as image_file: img_data = image_file.read() img_b64_str = base64.b64encode(img_data).decode('utf-8') # Open the image file in binary mode response = client.chat.completions.create( model="gpt-4o-mini", messages=[ { "role": "user", "content": [ {"type": "text", "text": prompt}, { "type": "image_url", "image_url": {"url": f"data:{img_type};base64,{img_b64_str}"}, }, ], } ], ) # Extract the generated description from the response summary = response.choices[0].message.content print("Generated Description:\n", summary) ``` ### backend/openai_test.py ```python import os import openai from dotenv import load_dotenv import asyncio from uagents import Agent, Context, Model from utils.message_config import system_prompt, user_prompt, catheter_instruction load_dotenv() openai.api_key = os.getenv("OPENAI_API_KEY") class ContextPrompt(Model): context: str text: str class Response(Model): text: str agent = Agent( name="user", endpoint="http://localhost:8000/submit", ) print("uAgent created successfully: ", agent.address) AI_AGENT_ADDRESS = agent.address prompt = ContextPrompt( context=system_prompt+user_prompt, text=catheter_instruction, ) # Function to make synchronous OpenAI API call in an asynchronous context async def get_openai_response(prompt_context, prompt_text): try: response = await asyncio.to_thread( openai.ChatCompletion.create, model="gpt-3.5-turbo", # Replace with "gpt-4" if you have access messages=[ { "role": "system", "content": "You are a helpful assistant that finds and fixes bugs in code.", }, {"role": "user", "content": f"{prompt_context}\n\n{prompt_text}"}, ], max_tokens=150, temperature=0.5, ) return response["choices"][0]["message"]["content"].strip() except Exception as e: print(f"Error during OpenAI API call: {e}") return None # Send message and make request to OpenAI API @agent.on_event("startup") async def send_message(ctx: Context): try: print("Sending request to OpenAI...") # Get response from OpenAI asynchronously text_response = await get_openai_response(prompt.context, prompt.text) if text_response: # Manually trigger handle_response function with the OpenAI response await handle_response(ctx, AI_AGENT_ADDRESS, Response(text=text_response)) print("Request sent and response received from OpenAI.") else: print("Failed to get a response from OpenAI.") except Exception as e: print(f"Error occurred while making request to OpenAI: {e}") # Handle OpenAI API response @agent.on_message(Response) async def handle_response(ctx: Context, sender: str, msg: Response): try: print("Receiving response from OpenAI...") ctx.logger.info(f"Received response from {sender}: {msg.text}") except Exception as e: print(f"Error occurred while handling response: {e}") if __name__ == "__main__": agent.run() ``` ### image_descrip_test/image_agent.py ```python from uagents import Agent, Context from models.models import ImagePrompt, ImageResponse from uagents.setup import fund_agent_if_low from utils.openai_api import get_openai_response from utils.config import setup_logging import base64 import io import torch from PIL import Image from transformers import BlipProcessor, BlipForConditionalGeneration logger = setup_logging() # Initialize the agent image_agent = Agent( name="image_agent", seed="image_agent_seed_unique", port=8002, endpoint="http://localhost:8002/submit", ) # Fund the agent if the balance is low fund_agent_if_low(image_agent.wallet.address()) # Load BLIP model and processor during startup @image_agent.on_event("startup") async def startup_event(ctx: Context): try: logger.info(f"Image Agent Address: {image_agent.address}") # Load BLIP model and processor ctx.storage.set( "processor", BlipProcessor.from_pretrained('Salesforce/blip-image-captioning-base') ) ctx.storage.set( "model", BlipForConditionalGeneration.from_pretrained('Salesforce/blip-image-captioning-base') ) logger.info("BLIP model and processor loaded successfully.") except Exception as e: logger.error(f"Error during startup: {e}") # Handle incoming messages with image data @image_agent.on_message(ImagePrompt) async def handle_image_message(ctx: Context, sender: str, msg: ImagePrompt): try: # Decode the base64-encoded image data image_data = base64.b64decode(msg.image_data) image = Image.open(io.BytesIO(image_data)).convert('RGB') # Retrieve the BLIP model and processor from storage processor = ctx.storage.get("processor") model = ctx.storage.get("model") # Generate caption using BLIP inputs = processor(image, return_tensors='pt') with torch.no_grad(): out = model.generate(**inputs, max_length=50) caption = processor.decode(out[0], skip_special_tokens=True) logger.info(f"Generated Caption: {caption}") # Prepare prompt for in-depth summary prompt_text = ( f"Provide a thorough and detailed analysis, including possible contexts, emotions, and implications, " f"based on the following image description:\n\n'{caption}'" ) # Get in-depth summary from OpenAI in_depth_summary = await get_openai_response( prompt_context="", prompt_text=prompt_text, max_tokens=500, temperature=0.7 ) # Send the response back to the user await ctx.send(sender, ImageResponse( caption=caption, summary=in_depth_summary )) logger.info("In-depth summary sent to the user.") except Exception as e: logger.error(f"Error in image agent: {e}") await ctx.send(sender, ImageResponse( caption="", summary="An error occurred while processing your image. Please try again later." )) # Run the agent if __name__ == "__main__": image_agent.run() ``` [11 more indexed source files omitted to keep this export small. The full file list is in the Codebase structure section above.]