Scam Likely Detected!
'; }; // Function to close the modal const closeModal = () => { modal.style.display = 'none'; }; // Add event listener for the close button on the modal closeBtn.addEventListener('click', closeModal); // Start recording when the startRecordingBtn is clicked // Add event listener to start recording when the button is clicked startRecordingBtn.addEventListener('click', async () => { // Your recording logic here if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) { alert('Microphone access is not supported by this browser.'); return; } try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); mediaRecorder = new MediaRecorder(stream); audioChunks = []; mediaRecorder.ondataavailable = event => { audioChunks.push(event.data); }; mediaRecorder.start(); } catch (err) { console.error('Error accessing the microphone', err); } }); // Stop recording and process the audio when the stopRecordingBtn is clicked stopRecordingBtn.addEventListener('click', () => { if (mediaRecorder) { mediaRecorder.stop(); mediaRecorder.onstop = async () => { const audioBlob = new Blob(audioChunks, { type: 'audio/wav' }); await sendAudioToServer(audioBlob); }; } }); // Send audio to the server for transcription with Whisper async function sendAudioToServer(audioBlob) { // Convert Blob to base64 as OpenAI's API might require the audio in base64 format const reader = new FileReader(); reader.readAsDataURL(audioBlob); reader.onloadend = async () => { const base64Audio = reader.result.split(',')[1]; // Remove the Data URL part const OPENAI_API_URL = 'https://api.openai.com/v1/whisper'; // Hypothetical URL const OPENAI_API_KEY = process.env.OPENAI_API_KEY; // Securely manage this token try { const response = await fetch(OPENAI_API_URL, { method: 'POST', headers: { 'Authorization': `Bearer ${OPENAI_API_KEY}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ model: "whisper-1", // Specify the model you wish to use audio: base64Audio, }), }); if (!response.ok) { console.error('Failed to transcribe audio with Whisper'); return; } const { text } = await response.json(); // Adjust based on the actual API response console.log(text); // Do something with the transcription } catch (error) { console.error('Error transcribing audio with Whisper:', error); } }; } // Check the transcription for potential scams async function checkForScams(transcribedText) { const scamResult = await queryScamLLM(transcribedText); if (scamResult && scamResult.length > 0 && scamResult[0].label === "LABEL_1" && scamResult[0].score > 0.6) { showScamModal(); } } }); ``` ### transcribe_file.py ```python import argparse import os import numpy as np import speech_recognition as sr import whisper import torch from datetime import datetime, timedelta from queue import Queue from transformers import pipeline from time import sleep from sys import platform from pydub import AudioSegment import simpleaudio as sa def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", default="medium", help="Model to use", choices=["tiny", "base", "small", "medium", "large"]) parser.add_argument("--non_english", action='store_true', help="Don't use the english model.") parser.add_argument("--energy_threshold", default=1000, help="Energy level for mic to detect.", type=int) parser.add_argument("--record_timeout", default=2, help="How real time the recording is in seconds.", type=float) parser.add_argument("--phrase_timeout", default=1, help="How much empty space between recordings before we " "consider it a new line in the transcription.", type=float) parser.add_argument("--file", default=None, type=str, help="audiofile to transcribe") args = parser.parse_args() # The last time a recording was retrieved from the queue. phrase_time = None # Thread safe Queue for passing data from the threaded recording callback. data_queue = Queue() # We use SpeechRecognizer to record our audio because it has a nice feature where it can detect when speech ends. recorder = sr.Recognizer() recorder.energy_threshold = args.energy_threshold # Definitely do this, dynamic energy compensation lowers the energy threshold dramatically to a point where the SpeechRecognizer never stops recording. recorder.dynamic_energy_threshold = False # Load / Download model model = args.model if args.model != "large" and not args.non_english: model = model + ".en" audio_model = whisper.load_model(model) record_timeout = args.record_timeout phrase_timeout = args.phrase_timeout scam_model = pipeline(task="text-classification", model="phishbot/ScamLLM", top_k=None) transcription = [''] audio = AudioSegment.from_mp3(args.file) chunk_length_ms = 2 * 1000 # 30 seconds in milliseconds chunks = [audio[i:i + chunk_length_ms] for i in range(0, len(audio), chunk_length_ms)] print("Model loaded.") now = datetime.utcnow() for i, chunk in enumerate(chunks): print(i) source = chunk.export('temp_chunk.wav', format="wav") with sr.AudioFile('temp_chunk.wav') as source: # play_obj = sa.play_buffer(source.read(), num_channels=1, bytes_per_sample=2, sample_rate=44100) recorder.adjust_for_ambient_noise(source) audio_data = recorder.record(source) try: phrase_complete = False # If enough time has passed between recordings, consider the phrase complete. # Clear the current working audio buffer to start over with the new data. if phrase_time and now - phrase_time > timedelta(seconds=phrase_timeout): phrase_complete = True # This is the last time we received new audio data from the queue. phrase_time = now audio_np = np.frombuffer(audio_data.get_raw_data(), dtype=np.int16).astype(np.float32) / 32768.0 result = audio_model.transcribe(audio_np, fp16=torch.cuda.is_available()) text = result['text'].strip() if text and phrase_complete: transcription.append(text) elif text: transcription[-1] = text # Clear the console to reprint the updated transcription. os.system('cls' if os.name=='nt' else 'clear') for line in transcription: print(line) # Flush stdout. print('', end='', flush=True) # Infinite loops are bad for processors, must sleep. is_scam = scam_model(".".join(transcription)) for dictionary in is_scam[0]: if dictionary['label'] == "LABEL_1": print("Is scam: ", dictionary['score']) if dictionary['score'] > 0.6: print("Scam detected") sleep(0.25) except sr.UnknownValueError: print(f"Chunk {i} could not be understood.") except sr.RequestError as e: print(f"Could not request results; {e}") print("\n\nTranscription:") for line in transcription: print(line) if __name__ == "__main__": main() ``` ### transcribe_demo.py ```python import argparse import os import numpy as np import speech_recognition as sr import whisper import torch from datetime import datetime, timedelta from queue import Queue from transformers import AutoTokenizer, AutoModelForSequenceClassification from time import sleep from sys import platform def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", default="medium", help="Model to use", choices=["tiny", "base", "small", "medium", "large"]) parser.add_argument("--non_english", action='store_true', help="Don't use the english model.") parser.add_argument("--energy_threshold", default=1000, help="Energy level for mic to detect.", type=int) parser.add_argument("--record_timeout", default=2, help="How real time the recording is in seconds.", type=float) parser.add_argument("--phrase_timeout", default=1, help="How much empty space between recordings before we " "consider it a new line in the transcription.", type=float) parser.add_argument("--file", default=None, type=str, help="audiofile to transcribe") if 'linux' in platform: parser.add_argument("--default_microphone", default='pulse', help="Default microphone name for SpeechRecognition. " "Run this with 'list' to view available Microphones.", type=str) args = parser.parse_args() # The last time a recording was retrieved from the queue. phrase_time = None data_queue = Queue() recorder = sr.Recognizer() recorder.energy_threshold = args.energy_threshold recorder.dynamic_energy_threshold = False # if 'linux' in platform: mic_name = "MacBook Pro Microphone" if not mic_name or mic_name == 'list': print("Available microphone devices are: ") for index, name in enumerate(sr.Microphone.list_microphone_names()): print(f"Microphone with name \"{name}\" found") return else: for index, name in enumerate(sr.Microphone.list_microphone_names()): if mic_name in name: source = sr.Microphone(sample_rate=16000, device_index=index) break # Load / Download model model = args.model if args.model != "large" and not args.non_english: model = model + ".en" audio_model = whisper.load_model(model) record_timeout = args.record_timeout phrase_timeout = args.phrase_timeout tokenizer = AutoTokenizer.from_pretrained("phishbot/ScamLLM") scam_model = AutoModelForSequenceClassification.from_pretrained("phishbot/ScamLLM") transcription = [''] with source: recorder.adjust_for_ambient_noise(source) def record_callback(_, audio:sr.AudioData) -> None: """ Threaded callback function to receive audio data when recordings finish. audio: An AudioData containing the recorded bytes. """ # Grab the raw bytes and push it into the thread safe queue. data = audio.get_raw_data() data_queue.put(data) # Create a background thread that will pass us raw audio bytes. # We could do this manually but SpeechRecognizer provides a nice helper. recorder.listen_in_background(source, record_callback, phrase_time_limit=record_timeout) # Cue the user that we're ready to go. print("Model loaded.") print(source.device_index) while True: try: now = datetime.utcnow() # Pull raw recorded audio from the queue. if not data_queue.empty(): phrase_complete = False # If enough time has passed between recordings, consider the phrase complete. # Clear the current working audio buffer to start over with the new data. if phrase_time and now - phrase_time > timedelta(seconds=phrase_timeout): phrase_complete = True # This is the last time we received new audio data from the queue. phrase_time = now # Combine audio data from queue audio_data = b''.join(data_queue.queue) data_queue.queue.clear() # Convert in-ram buffer to something the model can use directly without needing a temp file. # Convert data from 16 bit wide integers to floating point with a width of 32 bits. # Clamp the audio stream frequency to a PCM wavelength compatible default of 32768hz max. audio_np = np.frombuffer(audio_data, dtype=np.int16).astype(np.float32) / 32768.0 # Read the transcription. result = audio_model.transcribe(audio_np, fp16=torch.cuda.is_available()) text = result['text'].strip() # If we detected a pause between recordings, add a new item to our transcription. # Otherwise edit the existing one. if text and phrase_complete: transcription.append(text) elif text: transcription[-1] = text # Clear the console to reprint the updated transcription. os.system('cls' if os.name=='nt' else 'clear') for line in transcription: print(line) # Flush stdout. print('', end='', flush=True) # Infinite loops are bad for processors, must sleep. is_scam = scam_model(".".join(transcription)) for dictionary in is_scam[0]: if dictionary['label'] == "LABEL_1": print("Is scam: ", dictionary['score']) if dictionary['score'] > 0.6: print("Scam detected") sleep(0.25) e [truncated — 167 more characters] ``` ### streamlit_test.py ```python import streamlit as st import numpy as np from streamlit_webrtc import WebRtcMode, webrtc_streamer # from streamlit_webrtc import VideoTransformerBase, VideoTransformerContext from pydub import AudioSegment import queue, pydub, tempfile, whisper, os, time import torch model = "base" audio_model = whisper.load_model(model) def save_audio(audio_segment: AudioSegment, base_filename: str) -> None: """ Save an audio segment to a .wav file. Args: audio_segment (AudioSegment): The audio segment to be saved. base_filename (str): The base filename to use for the saved .wav file. """ filename = f"{base_filename}_{int(time.time())}.wav" audio_segment.export(filename, format="wav") def transcribe(audio_segment: AudioSegment, debug: bool = False) -> str: """ Transcribe an audio segment using OpenAI's Whisper ASR system. Args: audio_segment (AudioSegment): The audio segment to transcribe. debug (bool): If True, save the audio segment for debugging purposes. Returns: str: The transcribed text. """ if debug: save_audio(audio_segment, "debug_audio") audio_np = np.frombuffer(audio_segment.raw_data, np.int16).flatten().astype(np.float32) / 32768.0 result = audio_model.transcribe(audio_np, fp16=False) text = result['text'].strip() # with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmpfile: # audio_segment.export(tmpfile.name, format="wav") # answer = openai.Audio.transcribe( # "whisper-1", # tmpfile, # temperature=0.2, # prompt="", # )["text"] # tmpfile.close() # os.remove(tmpfile.name) return text def frame_energy(frame): """ Compute the energy of an audio frame. Args: frame (VideoTransformerBase.Frame): The audio frame to compute the energy of. Returns: float: The energy of the frame. """ samples = np.frombuffer(frame.to_ndarray().tobytes(), dtype=np.int16) return np.sqrt(np.mean(samples**2)) def process_audio_frames(audio_frames, sound_chunk, silence_frames, energy_threshold): """ Process a list of audio frames. Args: audio_frames (list[VideoTransformerBase.Frame]): The list of audio frames to process. sound_chunk (AudioSegment): The current sound chunk. silence_frames (int): The current number of silence frames. energy_threshold (int): The energy threshold to use for silence detection. Returns: tuple[AudioSegment, int]: The updated sound chunk and number of silence frames. """ for audio_frame in audio_frames: sound_chunk = add_frame_to_chunk(audio_frame, sound_chunk) energy = frame_energy(audio_frame) if energy < energy_threshold: silence_frames += 1 else: silence_frames = 0 return sound_chunk, silence_frames def add_frame_to_chunk(audio_frame, sound_chunk): """ Add an audio frame to a sound chunk. Args: audio_frame (VideoTransformerBase.Frame): The audio frame to add. sound_chunk (AudioSegment): The current sound chunk. Returns: AudioSegment: The updated sound chunk. """ sound = pydub.AudioSegment( data=audio_frame.to_ndarray().tobytes(), sample_width=audio_frame.format.bytes, frame_rate=audio_frame.sample_rate, channels=len(audio_frame.layout.channels), ) sound_chunk += sound return sound_chunk def handle_silence(sound_chunk, silence_frames, silence_frames_threshold, text_output): """ Handle silence in the audio stream. Args: sound_chunk (AudioSegment): The current sound chunk. silence_frames (int): The current number of silence frames. silence_frames_threshold (int): The silence frames threshold. text_output (st.empty): The Streamlit text output object. Returns: tuple[AudioSegment, int]: The updated sound chunk and number of silence frames. """ if silence_frames >= silence_frames_threshold: if len(sound_chunk) > 0: text = transcribe(sound_chunk) text_output.write(text) sound_chunk = pydub.AudioSegment.empty() silence_frames = 0 return sound_chunk, silence_frames def handle_queue_empty(sound_chunk, text_output): """ Handle the case where the audio frame queue is empty. Args: sound_chunk (AudioSegment): The current sound chunk. text_output (st.empty): The Streamlit text output object. Returns: AudioSegment: The updated sound chunk. """ if len(sound_chunk) > 0: text = transcribe(sound_chunk) text_output.write(text) sound_chunk = pydub.AudioSegment.empty() return sound_chunk def app_sst( status_indicator, text_output, timeout=3, energy_threshold=2000, silence_frames_threshold=100 ): """ The main application function for real-time speech-to-text. This function creates a WebRTC streamer, starts receiving audio data, processes the audio frames, and transcribes the audio into text when there is silence longer than a certain threshold. Args: status_indicator: A Streamlit object for showing the status (running or stopping). text_output: A Streamlit object for showing the transcribed text. timeout (int, optional): Timeout for getting frames from the audio receiver. Default is 3 seconds. energy_threshold (int, optional): The energy threshold below which a frame is considered silence. Default is 2000. silence_frames_threshold (int, optional): The number of consecutive silence frames to trigger transcription. Default is 100 frames. """ webrtc_ctx = webrtc_streamer( key="speech-to-text", mode=WebRtcMode.SENDONLY, audio_receiver_size=1024, media_stream_constraints={"video [truncated — 1173 more characters] ``` ### main_llm.py ```python import argparse import os import numpy as np import speech_recognition as sr import whisper import torch from datetime import datetime, timedelta from queue import Queue from transformers import pipeline from time import sleep from sys import platform import streamlit as st def main(): parser = argparse.ArgumentParser() parser.add_argument("--model", default="medium", help="Model to use", choices=["tiny", "base", "small", "medium", "large"]) parser.add_argument("--non_english", action='store_true', help="Don't use the english model.") parser.add_argument("--energy_threshold", default=1000, help="Energy level for mic to detect.", type=int) parser.add_argument("--record_timeout", default=2, help="How real time the recording is in seconds.", type=float) parser.add_argument("--phrase_timeout", default=1, help="How much empty space between recordings before we " "consider it a new line in the transcription.", type=float) parser.add_argument("--file", default=None, type=str, help="audiofile to transcribe") if 'linux' in platform: parser.add_argument("--default_microphone", default='pulse', help="Default microphone name for SpeechRecognition. " "Run this with 'list' to view available Microphones.", type=str) args = parser.parse_args() # The last time a recording was retrieved from the queue. phrase_time = None data_queue = Queue() recorder = sr.Recognizer() recorder.energy_threshold = args.energy_threshold recorder.dynamic_energy_threshold = False # if 'linux' in platform: mic_name = "MacBook Pro Microphone" if not mic_name or mic_name == 'list': print("Available microphone devices are: ") for index, name in enumerate(sr.Microphone.list_microphone_names()): print(f"Microphone with name \"{name}\" found") return else: for index, name in enumerate(sr.Microphone.list_microphone_names()): if mic_name in name: source = sr.Microphone(sample_rate=16000, device_index=index) break if 'clicked' not in st.session_state: st.session_state['clicked'] = False # Load / Download model model = args.model if args.model != "large" and not args.non_english: model = model + ".en" audio_model = whisper.load_model(model) record_timeout = args.record_timeout phrase_timeout = args.phrase_timeout scam_model = pipeline(task="text-classification", model="phishbot/ScamLLM", top_k=None) transcription = [''] with source: recorder.adjust_for_ambient_noise(source) def record_callback(_, audio:sr.AudioData) -> None: """ Threaded callback function to receive audio data when recordings finish. audio: An AudioData containing the recorded bytes. """ # Grab the raw bytes and push it into the thread safe queue. data = audio.get_raw_data() data_queue.put(data) # Create a background thread that will pass us raw audio bytes. # We could do this manually but SpeechRecognizer provides a nice helper. recorder.listen_in_background(source, record_callback, phrase_time_limit=record_timeout) # Cue the user that we're ready to go. print("Model loaded.") print(source.device_index) st.markdown("# Geras") st.markdown("Geras is a AI-powered tool that prevents the elderly from being scammed. We use OpenAI's Whisper API to transcribe the audio and another text classification LLM to detect if the person is being scammed.") st.markdown("Let's give it a try. Please click the button below to record.") # print_info = False refreshable_box = st.empty() def record_button(): st.write("Recording... Text below:") st.session_state.clicked = True break_loop = False def stop_button(): st.write("Stopped recording.") break_loop = True st.session_state.clicked = False refreshable_box.empty() return st.button('Record', on_click=record_button) st.button("Stop", on_click=stop_button) while True: try: if break_loop: break now = datetime.utcnow() # Pull raw recorded audio from the queue. if not data_queue.empty(): phrase_complete = False # If enough time has passed between recordings, consider the phrase complete. # Clear the current working audio buffer to start over with the new data. if phrase_time and now - phrase_time > timedelta(seconds=phrase_timeout): phrase_complete = True # This is the last time we received new audio data from the queue. phrase_time = now # Combine audio data from queue audio_data = b''.join(data_queue.queue) data_queue.queue.clear() # Convert in-ram buffer to something the model can use directly without needing a temp file. # Convert data from 16 bit wide integers to floating point with a width of 32 bits. # Clamp the audio stream frequency to a PCM wavelength compatible default of 32768hz max. audio_np = np.frombuffer(audio_data, dtype=np.int16).astype(np.float32) / 32768.0 # Read the transcription. result = audio_model.transcribe(audio_np, fp16=torch.cuda.is_available()) text = result['text'].strip() # If we detected a pause between recordings, add a new item to our transcription. # Otherwise edit the existing one. if st. [truncated — 1425 more characters] ``` [2 more indexed source files omitted to keep this export small. The full file list is in the Codebase structure section above.]