I am using a fish eye lens for object detection to maximize FOV. I have code to undistort the image but the reprojection error (2.9) is not great and the edges are really messed up. I have attached some of my code and an image of the undistortion. For calibration, I took a 8 by 11 checkerboard (7x10 inner corners) and took many images in each plane.
def calibrate(showPics=True): # Read Image root = os.getcwd() calibrationDir = os.path.join(root,'snapshots//') imgPathList = glob.glob(os.path.join(calibrationDir,'*.jpg')) # Initialize nRows = 7 nCols = 10 termCriteria = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_MAX_ITER,30,0.001) worldPtsCur = np.zeros((nRows*nCols,3), np.float32) worldPtsCur[:,:2] = np.mgrid[0:nRows,0:nCols].T.reshape(-1,2) worldPtsList = [] imgPtsList = [] # Find Corners for curImgPath in imgPathList: imgBGR = cv.imread(curImgPath) imgGray = cv.cvtColor(imgBGR, cv.COLOR_BGR2GRAY) cornersFound, cornersOrg = cv.findChessboardCorners(imgGray,(nRows,nCols), None) if cornersFound == True: worldPtsList.append(worldPtsCur) cornersRefined = cv.cornerSubPix(imgGray,cornersOrg,(11,11),(-1,-1),termCriteria) imgPtsList.append(cornersRefined) if showPics: cv.drawChessboardCorners(imgBGR,(nRows,nCols),cornersRefined,cornersFound) cv.imshow('Chessboard', imgBGR) cv.waitKey(500) cv.destroyAllWindows() # Calibrate repError,camMatrix,distCoeff,rvecs,tvecs = cv.calibrateCamera(worldPtsList, imgPtsList, imgGray.shape[::-1],None,None) print('Camera Matrix:\n',camMatrix) print('distCoeff',distCoeff) print("Reproj Error (pixels): {:.4f}".format(repError)) return camMatrix,distCoeffdef removeDistortion(camMatrix,distCoeff): root = os.getcwd() imgPath = os.path.join(root,'snapshots//snapshot_20240310_153742.jpg') img = cv.imread(imgPath) height,width = img.shape[:2] camMatrixNew,roi = cv.getOptimalNewCameraMatrix(camMatrix,distCoeff,(width,height),1,(width,height)) imgUndist = cv.undistort(img,camMatrix,distCoeff,None,camMatrixNew)