If it's a standalone persistant connection, I have no problem, connection lasts for hours.
If I use psycopg(3) Connection pool, I make make requests and during a period of time I no problem. But at one point I get a Pool Timeout when client makes a new request.
Then I tried: start pool, do not request anything, just wait. After some time (around 1h) I look at postgresql (pg_stat_activity), I have 5 idle (=pool size) connections. Then I make a request from client, and all connections vanish at same time (I can see it from pg_stat_activity) and Pool Timeout, and situation is stuck.
I also tried to decrease max_timeout to 900 but still same issue.
def init_pool(self, min_cnx=5): cnx_str = f"host={DB_HOST} port={DB_PORT} dbname={DB_NAME} user={DB_USERNAME} password={DB_USERPWD}" self.pool = ConnectionPool(conninfo=cnx_str, min_size=min_cnx, open=True, check=ConnectionPool.check_connection)def query(self, q, dbv=None, debug=False) -> list: print("pool size: ", len(self.pool._pool)) print("pool stats before: ", self.pool.get_stats()) with self.pool.connection() as cnx: if cnx.closed: self.pool.check() raise ConnectionError("ERROR: PostgreSQL cnx from pool is closed.") cnx.autocommit = True cnx.row_factory = self.row_factory with psycopg.ClientCursor(cnx) as rdc: rdc.execute(q, dbv) if dbv else rdc.execute(q) if debug and rdc._query: print(rdc._query.query) if rdc.description: data = rdc.fetchall() else: data = [] print("pool stats after query: ", self.pool.get_stats()) print("pool stats after: ", self.pool.get_stats()) return dataAnd logs:
[pid: 236344|app: 0|req: 26/26] () {56 vars in 1083 bytes} [Mon Feb 5 11:41:56 2024] POST /v1/user => generated 933 bytes in109 msecs (HTTP/1.1 200) 8 headers in 749 bytes (1 switches on core 0)
pool size: 3
pool stats before: {'connections_num': 5, 'requests_num': 3, 'requests_queued': 1, 'connections_ms': 268, 'requests_wait_ms': 34, 'usage_ms': 34, 'pool_min': 5, 'pool_max': 5, 'pool_size': 5, 'pool_available': 3, 'requests_waiting': 0}
pool stats after query: {'connections_num': 5, 'requests_num': 4, 'requests_queued': 1, 'connections_ms': 268, 'requests_wait_ms': 34, 'usage_ms': 34, 'pool_min': 5, 'pool_max': 5, 'pool_size': 5, 'pool_available': 2, 'requests_waiting': 0}
pool stats after: {'connections_num': 5, 'requests_num': 4, 'requests_queued': 1, 'connections_ms': 268, 'requests_wait_ms': 34, 'usage_ms': 49, 'pool_min': 5, 'pool_max': 5, 'pool_size': 5, 'pool_available': 2, 'requests_waiting': 0}
[pid: 236344|app: 0|req: 28/28] () {56 vars in 1087 bytes} [Mon Feb 5 11:41:58 2024] POST /v1/iobjs => generated 4788 bytes in 29 msecs (HTTP/1.1 200) 6 headers in 302 bytes (1 switches on core 0)
[pid: 236344|app: 0|req: 29/29] () {54 vars in 816 bytes} [Mon Feb 5 11:42:05 2024] OPTIONS /v1/user/quit => generated 0 bytes in 0 msecs (HTTP/1.1 200) 6 headers in 307 bytes (0 switches on core 0)
pool size: 0
pool stats before: {'connections_num': 5, 'requests_num': 6, 'requests_queued': 1, 'connections_ms': 268, 'requests_wait_ms': 34, 'usage_ms': 62, 'pool_min': 5, 'pool_max': 5, 'pool_size': 5, 'pool_available': 0, 'requests_waiting': 0}
Traceback (most recent call last):
File "/var/srvr/log.py", line 68, in processself.db.query(File "/var/srvr/pg3p.py", line 71, in querywith self.pool.connection() as cnx:File "/usr/local/lib/python3.12/contextlib.py", line 137, in enterreturn next(self.gen)^^^^^^^^^^^^^^File "/var/srvr/lib/python3.12/site-packages/psycopg_pool/pool.py", line 170, in connectionconn = self.getconn(timeout=timeout)^^^^^^^^^^^^^^^^^^^^^^^^^^^^^File "/var/srvr/lib/python3.12/site-packages/psycopg_pool/pool.py", line 204, in getconnraise PoolTimeout(psycopg_pool.PoolTimeout: couldn't get a connection after 30.00 sec
pool size: 0
pool stats before: {'connections_num': 5, 'requests_num': 7, 'requests_queued': 2, 'connections_ms': 268, 'requests_wait_ms': 30035, 'usage_ms': 62, 'requests_errors': 1, 'pool_min': 5, 'pool_max': 5, 'pool_size': 5, 'pool_available': 0, 'requests_waiting': 1}