That's my first time of using selenium. The schema is pretty simple: there is, initially a page where there is a select, an input, another input and another select(which is not altered). The user(selenium) enters the information reading a txt line by line to put it on input(which is divided in 2 inputs). Then submit it by a click on a button. Then it goes to another page, it loads the page, extract the informations(I'm using BeautifulSoup for this). There is two bottons which says "show more" in (mostrar mais in Portuguese) or show less, which selenium has to click to show more informations and then BeautifulSoup extracts them. Then, these informations are sent to a csv file(which is created by the code itself). Selenium also takes screenshots of this second page.
I'll post the code, the error and the bit of the html of the page that matters:
from selenium import webdriver as driverfrom bs4 import BeautifulSoup as soupimport csvfrom selenium.webdriver.support.ui import Selectfrom selenium.webdriver.common.by import Byimport requestsfrom selenium.webdriver import FirefoxOptionsfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECclass Processo: def mostrarMaisInformacoes(self): ver_mais_cabecalho = WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.CLASS_NAME, "unj-link-collapse collapsed"))) ver_mais_cabecalho.click() ver_mais_movimentacoes = WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.CLASS_NAME, "linkNaoSelecionado unj-link-collapse"))) ver_mais_movimentacoes.click() def pegaElementos(self): self.mostrarMaisInformacoes() distribuicao_texto = "" controle_texto = "" valor_texto = "" numero_processo = soup.find("#numeroProcesso").get_text() classe_processo = soup.find("#classeProcesso").get_text() assunto_processo = soup.find("#assuntoProcesso").get_text() foro_processo = soup.find("#foroProcesso").get_text() vara_processo = soup.find("#varaProcesso").get_text() juiz_processo = soup.find("#juizProcesso").get_text() data_movimentacao_texto = "" requerente_texto = "" descricao_texto = "" outros_assuntos_texto = "" area_texto = "" distribuicao = soup.find("#dataHoraDistribuicaoProcesso") if len(distribuicao): distribuicao_texto = distribuicao.get_text() else: distribuicao_texto = "N/A" controle = soup.find("#numeroControleProcesso") if len(controle): controle_texto = controle.get_text() else: controle_texto = "N/A" valor_acao = soup.find("#valorAcaoProcesso") if len(valor_acao): valor_texto = valor_acao.get_text() else: valor_texto = "" outros_assuntos_coluna = soup.find(".col-lg-2 mb-2") if len(outros_assuntos_coluna): outros_assuntos_linha = soup.find(".line-clamp__2") outros_assuntos_texto = outros_assuntos_linha.find("span").get_text() else: outros_assuntos_texto = "N/A" #outros_assuntos area = soup.find("#areaProcesso") if len(area): area_texto = area.find(span).get_text() else: area_texto = "N/A" tabela_partes = soup.find("#tablePartesPrincipais") for requerente in tabela_partes.findAll("td", {"class": "nomeParteEAdvogado"}): requerente_texto = requerente.get_text() tabelas_movimentacoes = soup.find("#tabelaUltimasMovimentacoes") for dmovimentacao in tabelas_movimentacoes.findAll("td", {"class": "dataMovimentacao"}): data_movimentacao = dmovimentacao.get_text() for descricao in tabelas_movimentacoes.findAll("td", {"class": "descricaoMovimentacao"}): descricao_texto = descricao.get_text() data_list = [numero_processo, classe_processo, foro_processo, vara_processo, juiz_processo, assunto_processo,controle_texto, valor_texto, distribuicao_texto, area_texto, outros_assuntos_texto] return data_listif __name__ == '__main__': obj = Processo() opts = FirefoxOptions() opts.add_argument("--headless") driver = driver.Firefox(options = opts) url = "https://esaj.tjsp.jus.br/cpopg/open.do" driver.get(url) file1 = open('numerosprocesso.txt', 'r') Lines = file1.readlines() botao = WebDriverWait(driver, 50).until(EC.element_to_be_clickable((By.ID, "botaoConsultarProcessos"))) entrada_texto_primeiraparte = driver.find_element(By.ID, "numeroDigitoAnoUnificado") entrada_texto_segundaparte = driver.find_element(By.ID, "foroNumeroUnificado") elementos = [] for line in Lines: print(type(line)) seletor = driver.find_element(By.ID, "cbPesquisa") drop = Select(seletor) drop.select_by_value("NUMPROC") entrada_texto_primeiraparte.send_keys(line[0:14]) entrada_texto_segundaparte.send_keys(line[-4:]) botao.click() driver.implicitly_wait(20) obj.mostrarMaisInformacoes() elementos = obj.pegaElementos() driver.save_full_page_screenshot('processo'+ f'{line}.png') with open(csv_file, 'w', newline='', encoding='utf-8') as csvfile: csv_writer = csv.writer(csvfile) csv_writer.writerow(['Número do Processo', 'Classe', 'Foro', 'Vara', 'Juiz', 'Assunto', 'Controle','Valor', 'Distribuição','Área', 'Outros Assuntos']) for item in elementos: csv_writer.writerow([item[0], item[1], item[2], item[3], item[4], item[5], item[6], item[7], item[8], item[9], item[10]])here is the error... which is in the show more part the method is mostrarMaisInformacoes which means show More Informations
Traceback (most recent call last): File "processo.py", line 97, in <module> obj.mostrarMaisInformacoes() File "processo.py", line 14, in mostrarMaisInformacoes ver_mais_cabecalho = WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.CLASS_NAME, "unj-link-collapse collapsed"))) File "/home/andre/anaconda3/lib/python3.8/site-packages/selenium/webdriver/support/wait.py", line 105, in until raise TimeoutException(message, screen, stacktrace)selenium.common.exceptions.TimeoutException: Message: Stacktrace:RemoteError@chrome://remote/content/shared/RemoteError.sys.mjs:8:8WebDriverError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:189:5NoSuchElementError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:507:5dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:132:16and the relevant html:
div class="unj-entity-header__details__barra"><div class="container"><div class="unj-ta-r"><a href="#maisDetalhes" class="unj-link-collapse" data-toggle="collapse" aria-expanded="false" aria-controls="maisDetalhes"><span class="unj-link-collapse__show"><i id="botaoExpandirDadosSecundarios" class="unj-link-collapse__icon glyph glyph-chevron-down"></i> Mais</span><span class="unj-link-collapse__hide"><i id="botaoRecolherDadosSecundarios" class="unj-link-collapse__icon glyph glyph-chevron-up"></i> Recolher</span></a></div><div id="maisDetalhes" class="collapse" aria-expanded="false"><div class="row unj-row--border-top">