Selenium Python not rendering dynamic data - is there a hidden anti-automation code? - Stack Overflow

Here is my code:

import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdrivermon.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Set up ChromeDriver service
service = Service(r"C:\Users\Rob\Downloads\chromedriver-win64\chromedriver-win64\chromedriver.exe")  # Update path to ChromeDriver

# Set up Chrome options
chrome_options = Options()
chrome_options.add_argument("--start-maximized")  # Start browser maximized
chrome_options.add_argument("--disable-blink-features=AutomationControlled")  # Helps bypass bot detection
chrome_options.add_argument("--disable-infobars")  # Removes Chrome's automation banner

# Initialize WebDriver for Chrome
driver = webdriver.Chrome(service=service, options=chrome_options)

try:
    # Open the specified URL
    driver.get(",NYC-LON_2025-04-10&ad=1&yad=0&ch=0&inf=0&cabin=M&flex=LOWEST&ond=1")
    
    # Wait for the cookie consent button and click it
    try:
        cookie_button = WebDriverWait(driver, 15).until(
            EC.element_to_be_clickable((By.ID, "ensAcceptAll"))
        )
        cookie_button.click()
        print("Cookie consent accepted.")
    except Exception as e:
        print("Cookie consent button not found or not clickable:", e)

    # Optional: Add a delay to observe the page in the browser
    time.sleep(60)

finally:
    # Close the browser
    driver.quit()

I am using Selenium to download flight prices and options for a particular route through

/

I do not have access to the IAG Developer Tools API, and so am attempting to use web-scraping through a Selenium/Python program to achieve the same outcome.

The webpage that I am looking to simulate through Selenium is as follows:

,NYC-LON_2025-04-10&ad=1&yad=0&ch=0&inf=0&cabin=M&flex=LOWEST&ond=1

The above URL (the "Target URL") is the URL that I am redirected to when I visit the main booking page through

/

whereby I input a return trip from London to New York from 1 April 2025 to 10 April 2025, in Economy for one adult.

When I visit the Target URL, and use Developer Tools on Chrome to identify the Fetch/XHR requests in the Network tab, I have identified the below URL as the POST request which provides the flight information (prices and times) that I am looking for. This would be the information that I look to download via my program, once the webpage has rendered through Selenium.

;ondwanted=1?metasearchdata=LON_NYC_2025-04-01&metasearchdata=NYC_LON_2025-04-10&locale=en_GB

However, when I create a simple program using Selenium Python to access the Target URL, the website just never loads. A 'cookies' notification pops-up, which I 'accept' through Selenium, but no dynamic content seems to load. When I use the Developer Tools on the opened webpage (I am not using headless mode so I can debug easier), I don't see the above URL POST request that I am expecting to see.

Why is this not working? Is there perhaps some kind of automation prevention code that is embedded into the website which detects that I am accessing via Selenium, and prohibiting the dynamic data from being loaded?

I have even tried simply loading the following URL using Selenium, which renders fine:

/

And then inputted the search parameters I am looking for (a return trip from London to New York from 1 April 2025 to 10 April 2025, in Economy for one adult). However, the same issue remains (as I essentially get to the same Target URL, which doesn't render).

Any help would be greatly appreciated.

EDIT: To the extent helpful, here is the output on my Python console. I leave the webpage running for enough time to load, but it still just doesn't seem to load.

DevTools listening on ws://127.0.0.1:56232/devtools/browser/b219c155-3fa3-464e-9ed3-ba1330ad5f76

Cookie consent accepted.

Created TensorFlow Lite XNNPACK delegate for CPU.

Here is my code:

import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# Set up ChromeDriver service
service = Service(r"C:\Users\Rob\Downloads\chromedriver-win64\chromedriver-win64\chromedriver.exe")  # Update path to ChromeDriver

# Set up Chrome options
chrome_options = Options()
chrome_options.add_argument("--start-maximized")  # Start browser maximized
chrome_options.add_argument("--disable-blink-features=AutomationControlled")  # Helps bypass bot detection
chrome_options.add_argument("--disable-infobars")  # Removes Chrome's automation banner

# Initialize WebDriver for Chrome
driver = webdriver.Chrome(service=service, options=chrome_options)

try:
    # Open the specified URL
    driver.get("https://www.britishairways.com/travel/book/public/en_gb/flightList?onds=LON-NYC_2025-04-01,NYC-LON_2025-04-10&ad=1&yad=0&ch=0&inf=0&cabin=M&flex=LOWEST&ond=1")
    
    # Wait for the cookie consent button and click it
    try:
        cookie_button = WebDriverWait(driver, 15).until(
            EC.element_to_be_clickable((By.ID, "ensAcceptAll"))
        )
        cookie_button.click()
        print("Cookie consent accepted.")
    except Exception as e:
        print("Cookie consent button not found or not clickable:", e)

    # Optional: Add a delay to observe the page in the browser
    time.sleep(60)

finally:
    # Close the browser
    driver.quit()

I am using Selenium to download flight prices and options for a particular route through

https://www.britishairways.com/nx/b/en/gb/

I do not have access to the IAG Developer Tools API, and so am attempting to use web-scraping through a Selenium/Python program to achieve the same outcome.

The webpage that I am looking to simulate through Selenium is as follows:

https://www.britishairways.com/travel/book/public/en_gb/flightList?onds=LON-NYC_2025-04-01,NYC-LON_2025-04-10&ad=1&yad=0&ch=0&inf=0&cabin=M&flex=LOWEST&ond=1

The above URL (the "Target URL") is the URL that I am redirected to when I visit the main booking page through

https://www.britishairways.com/nx/b/en/gb/

whereby I input a return trip from London to New York from 1 April 2025 to 10 April 2025, in Economy for one adult.

When I visit the Target URL, and use Developer Tools on Chrome to identify the Fetch/XHR requests in the Network tab, I have identified the below URL as the POST request which provides the flight information (prices and times) that I am looking for. This would be the information that I look to download via my program, once the webpage has rendered through Selenium.

https://www.britishairways.com/api/sc4/badotcomadapter-paatwo/rs/v1/flightavailability/search;ondwanted=1?metasearchdata=LON_NYC_2025-04-01&metasearchdata=NYC_LON_2025-04-10&locale=en_GB

However, when I create a simple program using Selenium Python to access the Target URL, the website just never loads. A 'cookies' notification pops-up, which I 'accept' through Selenium, but no dynamic content seems to load. When I use the Developer Tools on the opened webpage (I am not using headless mode so I can debug easier), I don't see the above URL POST request that I am expecting to see.

Why is this not working? Is there perhaps some kind of automation prevention code that is embedded into the website which detects that I am accessing via Selenium, and prohibiting the dynamic data from being loaded?

I have even tried simply loading the following URL using Selenium, which renders fine:

https://www.britishairways.com/nx/b/en/gb/

And then inputted the search parameters I am looking for (a return trip from London to New York from 1 April 2025 to 10 April 2025, in Economy for one adult). However, the same issue remains (as I essentially get to the same Target URL, which doesn't render).

Any help would be greatly appreciated.

EDIT: To the extent helpful, here is the output on my Python console. I leave the webpage running for enough time to load, but it still just doesn't seem to load.

DevTools listening on ws://127.0.0.1:56232/devtools/browser/b219c155-3fa3-464e-9ed3-ba1330ad5f76

Cookie consent accepted.

Created TensorFlow Lite XNNPACK delegate for CPU.

• python
• selenium-webdriver
• dom

Share Improve this question Follow edited Jan 11 at 15:29 user296950 asked Jan 11 at 15:22 user296950user296950 3522 silver badges88 bronze badges 4

• Did you use ChatGPT or another AI to generate the code in question? – Barry the Platipus Commented Jan 11 at 15:29
• Hi! Yes exactly that. I'm not familiar with coding so have been using ChatGPT to assist with creating it. Is this where my issue lies? – user296950 Commented Jan 11 at 15:29
• Are you sure that the request is not there? Note that devtools only show requests initiated after they have been opened. So maybe you need to refresh the page to see all requests. Regarding your question, it is possible that they have some scraping prevention mechanism. This would make sense for this page in particular as they might not want to give easy access to all their prices. But this will be quite hard to find out as their code is obscured. – JSON Derulo Commented Jan 11 at 15:40
• Thanks for the response. I don't believe the request is there, even after I refresh the page and watch all requests on the Network tab. For example, here is the requests when I do it under a normal browser i.imgur.com/Vh9S8w7.png (item 12 is the request I'm looking for), and i.imgur.com/g8Yg1b0.png is the list of requests when running through Selenium, even after I have refreshed. Is there anything obvious that might be causing this in my code? Or is there an alternative to Selenium that could be a workaround? – user296950 Commented Jan 11 at 15:47

Add a comment  | 

1 Answer 1

Sorted by: Reset to default 0

Hi there I found luck building it up from the beginning from a slightly different BA website:

import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys

# Set up ChromeDriver service
service = Service(r"[Directory]\chromedriver-win64\chromedriver.exe")  # Update path to ChromeDriver

# Set up Chrome options
chrome_options = Options()
chrome_options.add_argument("--start-maximized")  # Start browser maximized
chrome_options.add_argument("--disable-blink-features=AutomationControlled")  # Helps bypass bot detection
chrome_options.add_argument("--disable-infobars")  # Removes Chrome's automation banner

# Initialize WebDriver for Chrome
driver = webdriver.Chrome(service=service, options=chrome_options)

try:
    #Open the specified URL
    driver.get("https://www.britishairways.com/nx/b/en/gb/")
    try:
        time.sleep(10)
        cookie_button = WebDriverWait(driver, 15).until(
            EC.element_to_be_clickable((By.ID, "ensAcceptAll"))
        )
        cookie_button.click()
        print("Cookie consent accepted.")
        time.sleep(5)
        
        date_pickerDepart = driver.find_element(By.XPATH, "/html/body/div[1]/section/main/section[6]/section/section/form/section[1]/section[3]/section[1]/div/input")
        date_pickerDepart.send_keys("01/04/2025")#Departure Date
        date_pickerDepart.submit()
        
        date_pickerReturn = driver.find_element(By.XPATH, "/html/body/div[1]/section/main/section[6]/section/section/form/section[1]/section[3]/section[2]/div/input")
        date_pickerReturn.send_keys("10/04/2025")#ReturnDate
        date_pickerReturn.submit()
        
        From_Picker = driver.find_element(By.XPATH, "/html/body/div[1]/section/main/section[6]/section/section/form/section[1]/section[1]/div/input")
        From_Picker.click()
        time.sleep(1)
        From_Picker.send_keys("LHR")#London Heathrow
        time.sleep(3)#Give it time to load the airports
        From_Picker.send_keys(Keys.RETURN)#Mimic someone clicking enter on a keyboard
        From_Picker.submit()
        
        To_Picker = driver.find_element(By.XPATH, "/html/body/div[1]/section/main/section[6]/section/section/form/section[1]/section[2]/div/input")
        To_Picker.click()
        time.sleep(1)
        To_Picker.send_keys("NYC")#New York
        time.sleep(3)#Give it time to load the airports
        To_Picker.send_keys(Keys.RETURN)#Mimic someone clicking enter on a keyboard
        To_Picker.submit()
        
        time.sleep(5)
        FindFlight_button = WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.XPATH, "//*[@id='ba-input-typeahead-1']")))
        FindFlight_button.click()
        time.sleep(5)
        
    except Exception as e:
        print("Cookie consent button not found or not clickable:", e)

    # Optional: Add a delay to observe the page in the browser
    time.sleep(60)

finally:
    #Close the browser
    driver.quit()

Got to give some more time to load stuff but its worked twice now for me. Hope this helps you along.