I am creating a python program that will start subprocesses that will take long time working and during that time the main process should communicate with the subprocess to pause, stop, resume and get its status.For now I am using two queues of multiprocess.Queue() one called action_queue and result_queue then I am always looping on the action queue within the subprocess and whenever there is an action, I push its result to the result_queue that the main process is also looping on.I am also using multiprocessing.Value() to check if the subprocess completed all its tasks.
I am not sure if this is an efficient way to do this by looping forever, is a better way for better performance that I am unaware of?
Here is my code:
import multiprocessingimport uuidimport threadingimport timeclass Action(object): id = str(uuid.uuid4().hex)[:12] command = "" returnable = 0 result = Noneclass Scan: def __init__(self, run_id): self.run_id = run_id self.action_queue = None self.result_queue = None self.scan_completed_flag = None self.status = 'running' def start(self, action_queue, result_queue, scan_completed_flag): self.action_queue = action_queue self.result_queue = result_queue self.scan_completed_flag = scan_completed_flag t = threading.Thread(target=self.monitior_scan) t.start() self.handle_actions() def get_status(self): return self.status def pause_scan(self): self.status = 'paused' return True def stop_scan(self): self.status = 'stopped' return True def resume_scan(self): self.status = 'running' return True def handle_actions(self): while True: if self.action_queue.qsize(): action = self.action_queue.get() print('got action', action.command) action.result = self.__getattribute__(action.command)() if action.returnable: self.result_queue.put(action) def monitior_scan(self): start_time = time.time() while True: time.sleep(1) end_time = time.time() self.scan_completed_flag.value = end_time - start_time breakclass Scanner: def __init__(self): self.locking = threading.Lock() self.scans = dict() def start_scan(self, run_id): result_queue = multiprocessing.Queue() action_queue = multiprocessing.Queue() scan_completed_flag = multiprocessing.Value('f', 0) self.scans[run_id] = dict() self.scans[run_id]['result_queue'] = result_queue self.scans[run_id]['action_queue'] = action_queue self.scans[run_id]['completed_flag'] = scan_completed_flag scan = Scan(run_id) scan_proc = multiprocessing.Process(target = scan.start, args=(action_queue, result_queue, scan_completed_flag)) scan_proc.start() self.scans[run_id]['proc'] = scan_proc print('stated', scan) def do_action(self, run_id, command, returnable = 0): if run_id not in self.scans: return "Run Id does not exist" action = Action() action.command = command action.returnable = returnable self.scans[run_id]['action_queue'].put(action) if returnable: while True: if run_id not in self.scans: return "Invalid Run Id or Scan already completed" try: if self.scans[run_id]['result_queue'].qsize(): with self.lock: action_result = self.scans[run_id]['result_queue'].get() if action.id == action_result.id: return action_result.result else: with self.lock: self.scans[run_id]['result_queue'].put(action_result) except: pass def monitor(self): while True: try: for scan, props in self.scans.items(): if props['completed_flag'].value: print(f'{scan} completed after {props["completed_flag"].value}') if self.scans[scan]['proc'].is_alive(): print(scan, 'alive') self.scans[scan]['proc'].terminate() del self.scans[scan] except: continueif __name__ == '__main__': scanner = Scanner() scanner.start_scan('abc') scanner.start_scan('33') scanner.start_scan('34') scanner.start_scan('35') scanner.start_scan('36') print(scanner.do_action('abc', 'get_status', 1)) scanner.do_action('abc', 'pause_scan', 1) print(scanner.do_action('abc', 'get_status', 1)) scanner.do_action('abc', 'stop_scan') print(scanner.do_action('abc', 'get_status', 1)) scanner.do_action('abc', 'resume_scan') print(scanner.do_action('abc', 'get_status', 1)) print(scanner.do_action('34', 'get_status', 1)) print(scanner.do_action('35', 'get_status', 1)) scanner.do_action('36', 'pause_scan', 1) print(scanner.do_action('34', 'get_status', 1)) scanner.do_action('35', 'stop_scan') print(scanner.do_action('36', 'get_status', 1)) scanner.do_action('33', 'resume_scan') print(scanner.do_action('34', 'get_status', 1)) print(scanner.do_action('33', 'get_status', 1))