Quantcast
Channel: Active questions tagged python - Stack Overflow
Viewing all articles
Browse latest Browse all 16536

Tokenizing problem Python. How do I tokenize a sentence?

$
0
0

I want to make a problem that basically takes as an input for example a sentence like "I like Python"and after a tokenizing process the output is "I /n like /n Python". This is basically a process/piping Python problem. Thing is is that I think the solution is quite a simple but I cant understand the problem really. The sentences are elements of a tuple. We need to have 3 processes, a source process that basically provides us with the information from the tuple, a transformer process that reads everything that the source process writes in the anonymous pip, it tokenizes it and then writes the result in a FIFO. Lastly, there is the output process which reads everything in the named pipe and it prints it. Imports only os, sys and time modules. This is the code I made but I take I can take suggestions that involve different code as a base.

import osimport sysimport timeMESSAGES = (    b"I love Python programming",    b"Inter-process communication is important",    b"Pipes and FIFOs are used for communication",)def source_process(write_pipe):    try:        for message in MESSAGES:            write_pipe.write(message)            write_pipe.flush()            time.sleep(1)  # Delay for easy reading    finally:        write_pipe.close()def transformer_process(read_pipe, fifo_path):    try:        fifo_fd = os.open(fifo_path, os.O_WRONLY)        while True:            message = read_pipe.readline()            if not message:                break            words = message.split()            transformed_message = b'\n'.join(words)            try:                os.write(fifo_fd, transformed_message + b'\n')            except OSError as e:                if e.errno == 32:  # Broken pipe                    break    finally:        read_pipe.close()        os.close(fifo_fd)def output_process(fifo_path):    try:        fifo_fd = os.open(fifo_path, os.O_RDONLY)        while True:            message = os.read(fifo_fd, 1024)            if not message:                break            print(message.decode(), end='', flush=True)    finally:        os.close(fifo_fd)def main():    try:        fifo_path = "my_fifo"        os.mkfifo(fifo_path)        source_pipe_read, source_pipe_write = os.pipe()        transformer_pipe_read, transformer_pipe_write = os.pipe()        source_pid = os.fork()        if source_pid == 0:            os.close(source_pipe_read)            os.close(transformer_pipe_write)            source_process(os.fdopen(source_pipe_write, 'wb'))            sys.exit(0)        else:            os.close(source_pipe_write)        transformer_pid = os.fork()        if transformer_pid == 0:            os.close(source_pipe_write)            os.close(transformer_pipe_read)            transformer_process(os.fdopen(transformer_pipe_write, 'wb'), fifo_path)            sys.exit(0)        else:            os.close(transformer_pipe_write)        output_pid = os.fork()        if output_pid == 0:            os.close(transformer_pipe_read)            output_process(fifo_path)            sys.exit(0)        else:            os.close(transformer_pipe_read)        # Close remaining pipes in the parent process        os.close(source_pipe_read)        os.close(transformer_pipe_write)        # Wait for all child processes to finish        os.waitpid(source_pid, 0)        os.waitpid(transformer_pid, 0)        os.waitpid(output_pid, 0)    except Exception as e:        print("An error occurred:", e)    finally:        # Remove the named pipe        try:            os.unlink(fifo_path)        except FileNotFoundError:            passif __name__ == "__main__":    main()

Viewing all articles
Browse latest Browse all 16536

Trending Articles