I'm trying to automate data retrieval from this website - https://renewablesandchp.ofgem.gov.uk/Public/ReportViewer.aspx
I'm copying the headers and form data from the flow I see in the Networks tab when submitting the form manually (n.b. the page size is not set by default). I also grab the ASP.Net cookie that is returned in the get_session_cookies query.
I then use the following code to try and query the service which returns a HTML snippet that is used to update the page (the data I want is in this snippet).
import requestsfrom bs4 import BeautifulSoup as bsdef get_report_viewer_soup(session: requests.Session) -> bs:"""Also grabs the ASP.Net session cookie""" url = 'https://renewablesandchp.ofgem.gov.uk/Public/ReportViewer.aspx' params = {'ReportPath': '/DatawarehouseReports/CertificatesExternalPublicDataWarehouse','ReportVisibility': 1,'ReportCategory': 2 } r = session.get(url, params=params) r.raise_for_status() return bs(r.text, 'html.parser')def get_table(session: requests.Session, soup: bs): url = 'https://renewablesandchp.ofgem.gov.uk/Public/ReportViewer.aspx' params = {'ReportPath': '/DatawarehouseReports/CertificatesExternalPublicDataWarehouse','ReportVisibility': 1,'ReportCategory': 2 } headers = {'Accept': '*/*','Accept-Encoding': 'gzip, deflate, br','Accept-Language': 'en-US,en;q=0.9','Cache-Control': 'no-cache','Connection': 'keep-alive','Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8','Host': 'renewablesandchp.ofgem.gov.uk','Origin': 'https://renewablesandchp.ofgem.gov.uk','Referer': 'https://renewablesandchp.ofgem.gov.uk/Public/ReportViewer.aspx?ReportPath=/DatawarehouseReports/CertificatesExternalPublicDataWarehouse&ReportVisibility=1&ReportCategory=2','Sec-Ch-Ua': '"Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"','Sec-Ch-Ua-Mobile': '?0','Sec-Ch-Ua-Platform': '"macOS"','Sec-Fetch-Dest': 'empty','Sec-Fetch-Mode': 'cors','Sec-Fetch-Site': 'same-origin','User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36','X-Microsoftajax': 'Delta=true','X-Requested-With': 'XMLHttpRequest' } data = {"ReportViewer$ctl04$ctl03$txtValue": "REGO, RO","ReportViewer$ctl04$ctl03$divDropDown$ctl01$HiddenIndices": "0,1","ReportViewer$ctl04$ctl05$txtValue": "Aerothermal, Biodegradable, Biogas, Biomass, Biomass 50kW DNC or less, Biomass using an Advanced Conversion Technology, CHP Energy from Waste, Co-firing of Biomass with Fossil Fuel, Co-firing of Energy Crops, Filled Storage Hydro, Filled Storage System, Fuelled, Geopressure, Geothermal, Hydro, Hydro 20MW DNC or less, Hydro 50kW DNC or less, Hydro greater than 20MW DNC, Hydrothermal, Landfill Gas, Micro Hydro, Ocean Energy, Off-shore Wind, On-shore Wind, Photovoltaic, Photovoltaic 50kW DNC or less, Sewage Gas, Solar and On-shore Wind, Tidal Flow, Tidal Power, Waste using an Advanced Conversion Technology, Wave Power, Wind, Wind 50kW DNC or less","ReportViewer$ctl04$ctl05$divDropDown$ctl01$HiddenIndices": "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33","ReportViewer$ctl04$ctl07$txtValue": "N/A, NIRO, RO, ROS","ReportViewer$ctl04$ctl07$divDropDown$ctl01$HiddenIndices": "0,1,2,3","ReportViewer$ctl04$ctl09$txtValue": "AD, Advanced gasification, Biomass (e.g. Plant or animal matter), Biomass using an Advanced Conversion Technology, Co-firing of biomass, Co-firing of biomass with fossil fuel, Co-firing of energy crops, Co-firing of regular bioliquid, Dedicated biomass, Dedicated biomass - BL, Dedicated biomass with CHP, Dedicated biomass with CHP - BL, Dedicated energy crops, Dedicated energy crops with CHP, Electricity generated from landfill gas, Electricity generated from sewage gas, Energy from waste with CHP, High-range co-firing, Low range co-firing of relevant energy crop, Low-range co-firing, Mid-range co-firing, N/A, Standard gasification, Station conversion, Station conversion - BL, Unit conversion, Unspecified, Waste using an Advanced Conversion Technology","ReportViewer$ctl04$ctl09$divDropDown$ctl01$HiddenIndices": "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27","ReportViewer$ctl04$ctl11$txtValue": "England, Northern Ireland, Scotland, Wales","ReportViewer$ctl04$ctl11$divDropDown$ctl01$HiddenIndices": "2,4,5,7","ReportViewer$ctl04$ctl17$txtValue": "<ALL>","ReportViewer$ctl04$ctl17$divDropDown$ctl01$HiddenIndices": "0","ReportViewer$ctl04$ctl27$txtValue": "General, NFFO, AMO","ReportViewer$ctl04$ctl27$divDropDown$ctl01$HiddenIndices": "0,1,2","ReportViewer$ctl04$ctl31$txtValue": "Issued, Revoked, Retired, Redeemed, Expired","ReportViewer$ctl04$ctl31$divDropDown$ctl01$HiddenIndices": "0,1,2,3,4","ReportViewer$ctl04$ctl37$txtValue": "<ALL>","ReportViewer$ctl04$ctl37$divDropDown$ctl01$HiddenIndices": "0","ReportViewer$ctl04$ctl13$ddValue": "1","ReportViewer$ctl04$ctl19$ddValue": "4","ReportViewer$ctl04$ctl21$ddValue": "3","ReportViewer$ctl04$ctl23$ddValue": "3","ReportViewer$ctl04$ctl25$ddValue": "3","ReportViewer$ctl04$ctl39$ddValue": "1","ReportViewer$ctl04$ctl35$ReportViewer_ctl04_ctl35": "rbTrue","ReportViewer$ctl04$ctl15$ReportViewer_ctl04_ctl15": "rbTrue","ReportViewer$ctl04$ctl29$txtValue": "","ReportViewer$ctl04$ctl29$cbNull": "on","ReportViewer$ctl04$ctl33$txtValue": "","ReportViewer$ctl04$ctl33$cbNull": "on","__VIEWSTATEGENERATOR": "75CF6949","hdnCookieConsent": "","hdnCookieAcceptanceRefreshDate": "","hdnCookieAcceptanceRefreshDay": "","hdnCookieAcceptanceRefreshMonth": "","ReportViewer$ctl03$ctl00": "","ReportViewer$ctl03$ctl01": "","ReportViewer$ctl10": "ltr","ReportViewer$ctl11": "standards","ReportViewer$AsyncWait$HiddenCancelField": "False","ReportViewer$ToggleParam$store": "","ReportViewer$ToggleParam$collapse": "false","ReportViewer$ctl08$ClientClickedId": "","ReportViewer$ctl07$store": "","ReportViewer$ctl07$collapse": "false","ReportViewer$ctl09$ScrollPosition": "","ReportViewer$ctl09$ReportControl$ctl04": "100","__ASYNCPOST": "true","ReportViewer$ctl04$ctl00": "View Report","ScriptManager1": "ScriptManager1|ReportViewer$ctl04$ctl00","ReportViewer$ctl05$ctl00$CurrentPage": "1","ReportViewer$ctl09$VisibilityState$ctl00": "ReportPage" } view_state_and_event_validation_inputs = { input_elem['id']: input_elem['value'] for input_elem in soup.find_all('input', type='hidden') if 'id' in input_elem.attrs and input_elem['id'] in ['__VIEWSTATE', '__EVENTVALIDATION'] } data.update(view_state_and_event_validation_inputs) r = session.post(url, params=params, headers=headers, data=data) r.raise_for_status() return rwith requests.Session() as session: soup = get_report_viewer_soup(session) r = get_table(session, soup)r.textHowever, this returns Validation Errors for each of the elements in the form, e.g.
"NullValueText":"Null","PostBackOnChange":true,"RelativeDivId":null,"TextBoxDisabledClass":null,"TextBoxDisabledColor":"#ECE9D8","TextBoxEnabledClass":null,"TextBoxId":"ReportViewer_ctl04_ctl03_txtValue","TriggerPostBackScript":function(){__doPostBack('ReportViewer$ctl04$ctl03','');},"ValidationMessage":"Please enter a value for the parameter \u0027Scheme:\u0027. The parameter cannot be blank.","ValidatorIdList":[]}, null, null, $get("ReportViewer_ctl04_ctl03")); });
What additional configuration is required to query this service succesfully? Any help would be much appreciated.