import time import re import unicodedata from urllib.parse import urlparse import pandas as pd from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.chrome.options import Options from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.select import Select from selenium.common.exceptions import TimeoutException, StaleElementReferenceException INPUT_FILE = "uploaded_files/lg_prod_list1.xlsx" OUTPUT_FILE = "uploaded_files/updated_lg_prod_list.xlsx" PAGES = [ "https://www.laurageller.com/collections/best-sellers?page=1", "https://www.laurageller.com/collections/best-sellers?page=2", "https://www.laurageller.com/collections/best-sellers?page=3", ] CART_URL = "https://www.laurageller.com/cart" HOME_URL = "https://www.laurageller.com/" CHECKOUT_BUTTON_XPATH = ( "(//button[contains(translate(., 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'CHECKOUT') " "or contains(translate(., 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'CHECK OUT') " "or @name='checkout' or contains(translate(@id, 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'CHECKOUT')] " "| //a[contains(translate(., 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'CHECKOUT') " "or contains(translate(., 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'CHECK OUT')] " "| //input[@name='checkout' or contains(translate(@value, 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'CHECK OUT') " "or contains(translate(@value, 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'CHECKOUT')])[1]" ) options = Options() options.add_argument("--headless=new") options.add_argument("--window-size=1920,1080") options.add_argument("--disable-gpu") options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-blink-features=AutomationControlled") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option("useAutomationExtension", False) driver = webdriver.Chrome(options=options) driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})") wait = WebDriverWait(driver, 12) def accept_cookies_once() -> None: try: btn = WebDriverWait(driver, 5).until( EC.element_to_be_clickable( (By.XPATH, "//button[contains(., 'Accept') or contains(., 'Agree')]") ) ) btn.click() time.sleep(1) print(" Cookie banner accepted") except TimeoutException: pass def wait_for_page_ready(timeout: int = 12) -> None: try: WebDriverWait(driver, timeout).until( lambda d: d.execute_script("return document.readyState") == "complete" ) except TimeoutException: pass def ensure_storefront_context() -> None: # After checkout/cart flows, force storefront context before collection scraping. if "/checkouts/" in (driver.current_url or ""): driver.get(HOME_URL) wait_for_page_ready() accept_cookies_once() time.sleep(1) def wait_for_checkout_url(timeout: int = 18) -> str: try: WebDriverWait(driver, timeout).until(lambda d: "/checkouts/" in (d.current_url or "")) except TimeoutException: pass return driver.current_url if "/checkouts/" in (driver.current_url or "") else "" def normalize_text(value: str) -> str: text = unicodedata.normalize("NFKC", value or "") text = text.replace("\u00a0", " ").replace("\u200b", "").replace("\u2013", "-").replace("\u2014", "-") return " ".join(text.strip().lower().split()) def normalize_tokens(value: str) -> list[str]: return re.findall(r"[a-z0-9]+", normalize_text(value)) def title_matches(requested: str, card_title: str) -> bool: req_tokens = normalize_tokens(requested) card_tokens = normalize_tokens(card_title) if not req_tokens or not card_tokens: return False # Strict title matching: only exact normalized token sequence is accepted. return req_tokens == card_tokens def normalize_product_path(value: str) -> str: if not value: return "" try: path = urlparse(value).path except Exception: path = value path = (path or "").strip().rstrip("/") return path.lower() def looks_like_product_url(value: str) -> bool: raw = (value or "").strip().lower() return raw.startswith("http://") or raw.startswith("https://") or "/products/" in raw def card_matches_url(product_url: str, card) -> bool: wanted = normalize_product_path(product_url) if not wanted: return False self_href = card.get_attribute("href") or "" self_path = normalize_product_path(self_href) if self_path and (self_path == wanted or self_path.endswith(wanted) or wanted.endswith(self_path)): return True links = card.find_elements(By.CSS_SELECTOR, "a[href*='/products/']") for link in links: href = link.get_attribute("href") or "" card_path = normalize_product_path(href) if not card_path: continue if card_path == wanted: return True if card_path.endswith(wanted) or wanted.endswith(card_path): return True return False def robust_click(element) -> bool: try: driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", element) time.sleep(0.4) element.click() return True except Exception: try: driver.execute_script("arguments[0].click();", element) return True except Exception: return False def element_is_enabled(element) -> bool: try: disabled_attr = element.get_attribute("disabled") aria_disabled = (element.get_attribute("aria-disabled") or "").strip().lower() classes = (element.get_attribute("class") or "").lower() return not disabled_attr and aria_disabled != "true" and "disabled" not in classes except Exception: return False def find_best_visible_add_button(): candidates = driver.find_elements( By.XPATH, "//button[contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO BAG') " "or contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO CART')]", ) if not candidates: return None width = driver.execute_script("return window.innerWidth;") best = None best_score = -10**9 for btn in candidates: try: if not btn.is_displayed() or not element_is_enabled(btn): continue rect = driver.execute_script( "const r=arguments[0].getBoundingClientRect();" "return {x:r.x,y:r.y,w:r.width,h:r.height};", btn, ) if not rect or rect["w"] < 100 or rect["h"] < 30: continue # Prefer right-side drawer button and lower position. score = rect["x"] + (rect["y"] * 0.05) if rect["x"] > width * 0.55: score += 5000 if score > best_score: best_score = score best = btn except Exception: continue return best def cart_has_item() -> bool: try: wait.until( EC.any_of( EC.presence_of_element_located((By.CSS_SELECTOR, ".cart-item, [data-cart-item], .line-item")), EC.presence_of_element_located((By.XPATH, "//form[contains(@action, '/cart')]//input[contains(@name, 'updates')]")), ) ) empty_nodes = driver.find_elements( By.XPATH, "//*[contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), 'your cart is empty')]", ) return len(empty_nodes) == 0 except TimeoutException: return False def get_cart_quantity() -> int: driver.get(CART_URL) time.sleep(1.5) qty = 0 quantity_inputs = driver.find_elements( By.XPATH, "//input[contains(@name, 'updates') or contains(@name, 'quantity') or @type='number']", ) for inp in quantity_inputs: try: if not inp.is_displayed(): continue val = (inp.get_attribute("value") or "").strip() if val.isdigit(): qty += int(val) except Exception: continue if qty > 0: return qty rows = driver.find_elements(By.CSS_SELECTOR, ".cart-item, [data-cart-item], .line-item") return len([r for r in rows if r.is_displayed()]) def find_checkout_buttons() -> list: return driver.find_elements(By.XPATH, CHECKOUT_BUTTON_XPATH) def click_checkout_if_visible(max_attempts: int = 3) -> bool: for _ in range(max_attempts): buttons = find_checkout_buttons() if not buttons: return False for btn in buttons: try: if not btn.is_displayed(): continue except StaleElementReferenceException: break except Exception: continue if robust_click(btn): print(" Clicked CHECKOUT") return True time.sleep(0.3) return False def is_any_checkout_visible(max_attempts: int = 3) -> bool: for _ in range(max_attempts): buttons = find_checkout_buttons() if not buttons: return False stale_seen = False for btn in buttons: try: if btn.is_displayed(): return True except StaleElementReferenceException: stale_seen = True break except Exception: continue if not stale_seen: return False time.sleep(0.3) return False def get_card_title_candidates(card) -> list[str]: candidates: list[str] = [] for attr in ("aria-label", "title"): value = (card.get_attribute(attr) or "").strip() if value and len(value) > 3: candidates.append(value) elements = card.find_elements( By.CSS_SELECTOR, "h2, h3, .card__heading, .product-title, .card-information__text, a[href*='/products/']", ) for elem in elements: text = elem.text.strip() if text and len(text) > 3 and all(key not in text.upper() for key in ["SHOP NOW", "ADD TO BAG", "ADD TO CART"]): candidates.append(text) lines = [line.strip() for line in card.text.split("\n") if line.strip()] for line in lines: line_up = line.upper() if len(line) < 4: continue if "SHOP NOW" in line_up or "ADD TO BAG" in line_up or "ADD TO CART" in line_up: continue if "RS." in line_up or "$" in line_up: continue if any(ch.isalpha() for ch in line): candidates.append(line) # De-duplicate while preserving order seen = set() deduped = [] for item in candidates: norm = normalize_text(item) if not norm or norm in seen: continue seen.add(norm) deduped.append(item) return deduped def get_product_cards() -> list: cards = driver.find_elements( By.CSS_SELECTOR, "li.grid__item, .product-item, .card-wrapper, [data-product-id]", ) if cards: return cards fallback_cards = driver.find_elements( By.XPATH, "//a[contains(@href, '/products/')]/ancestor::li[contains(@class, 'grid__item')][1] | " "//a[contains(@href, '/products/')]/ancestor::*[contains(@class, 'product-item')][1] | " "//a[contains(@href, '/products/')]/ancestor::*[contains(@class, 'card-wrapper')][1] | " "//a[contains(@href, '/products/')]/ancestor::*[@data-product-id][1]", ) deduped = [] seen = set() for card in fallback_cards: if card.id in seen: continue seen.add(card.id) deduped.append(card) if deduped: return deduped # Fallback: derive card containers from visible CTA buttons when product links are absent. cta_elements = driver.find_elements( By.XPATH, "//button[contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'SHOP NOW') " "or contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO BAG') " "or contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO CART')] " "| //a[contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'SHOP NOW')]", ) cta_cards = [] seen_cta_cards = set() for cta in cta_elements: try: card = cta.find_element( By.XPATH, "ancestor::li[1] | ancestor::*[contains(@class, 'grid__item')][1] | " "ancestor::*[contains(@class, 'product-item')][1] | ancestor::*[contains(@class, 'card-wrapper')][1] | " "ancestor::*[contains(@class, 'card')][1] | ancestor::*[contains(@class, 'product')][1]", ) if card.id in seen_cta_cards: continue seen_cta_cards.add(card.id) cta_cards.append(card) except Exception: continue if cta_cards: return cta_cards # Last resort: treat product anchors as cards when storefront markup differs. product_links = driver.find_elements(By.CSS_SELECTOR, "a[href*='/products/']") if not product_links: product_links = driver.find_elements( By.XPATH, "//a[contains(translate(@href, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '/products/')]", ) links_deduped = [] href_seen = set() for link in product_links: href = (link.get_attribute("href") or "").strip() if not href or href in href_seen: continue href_seen.add(href) links_deduped.append(link) return links_deduped def warm_up_listing_page() -> None: try: driver.execute_script("window.scrollTo(0, document.body.scrollHeight * 0.55);") time.sleep(0.8) driver.execute_script("window.scrollTo(0, 0);") time.sleep(0.5) except Exception: pass def click_card_action_by_exact_title(title: str) -> str: try: clicked_label = driver.execute_script( """const targetTitle = arguments[0]; const norm = (s) => (s || "").replace(/\\s+/g, " ").trim().toLowerCase(); const target = norm(targetTitle); const isVisible = (el) => !!el && !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); const isEnabled = (el) => !el.disabled && (el.getAttribute("aria-disabled") || "").toLowerCase() !== "true"; const actionCandidates = Array.from( document.querySelectorAll("button,a,input[type='button'],input[type='submit'],[role='button']") ).filter((el) => { if (!isVisible(el) || !isEnabled(el)) return false; const label = norm(el.textContent || el.value || el.getAttribute("aria-label") || ""); return label.includes("shop now") || label.includes("add to bag") || label.includes("add to cart"); }); const ancestors = ["li", "article", ".grid__item", ".card-wrapper", ".card", ".product-item", ".product-card", "[data-product-id]", "section", "div"]; for (const action of actionCandidates) { const label = norm(action.textContent || action.value || action.getAttribute("aria-label") || ""); let container = null; for (const sel of ancestors) { container = action.closest(sel); if (container) break; } const scopeText = norm((container ? container.textContent : action.parentElement?.textContent) || ""); if (!scopeText) continue; if (!scopeText.includes(target)) continue; action.scrollIntoView({block: "center"}); action.click(); if (label.includes("shop now")) return "SHOP NOW"; return "ADD TO BAG"; } return ""; """, title, ) return (clicked_label or "").strip().upper() except Exception: return "" def find_and_open_product(title: str, product_url: str = "") -> tuple[bool, str]: ensure_storefront_context() for page_idx, page in enumerate(PAGES, start=1): print(f" Loading best-seller page {page_idx}: {page}") cards = [] for attempt in range(1, 4): driver.get(page) time.sleep(2.5) accept_cookies_once() warm_up_listing_page() try: WebDriverWait(driver, 8).until( EC.any_of( EC.presence_of_element_located((By.CSS_SELECTOR, "li.grid__item, .product-item, .card-wrapper, [data-product-id]")), EC.presence_of_element_located((By.CSS_SELECTOR, "a[href*='/products/']")), EC.presence_of_element_located( ( By.XPATH, "//button[contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'SHOP NOW') " "or contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO BAG') " "or contains(translate(normalize-space(.), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO CART')]", ) ), EC.presence_of_element_located( (By.XPATH, "//a[contains(translate(@href, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '/products/')]") ), ) ) except TimeoutException: pass cards = get_product_cards() if not cards: try: driver.refresh() wait_for_page_ready() time.sleep(1) cards = get_product_cards() except Exception: pass if not cards: clicked_label = click_card_action_by_exact_title(title) if clicked_label: print(f" Clicked action: {clicked_label}") time.sleep(3) return True, clicked_label if cards: break if attempt < 3: print(f" Product cards not loaded (attempt {attempt}/3), retrying...") time.sleep(2) print(f" Product cards found: {len(cards)}") for i, card in enumerate(cards, start=1): try: matched_by_url = card_matches_url(product_url, card) matched_title = "" if not matched_by_url: title_candidates = get_card_title_candidates(card) matched_title = next((t for t in title_candidates if title_matches(title, t)), "") if not matched_by_url and not matched_title: continue if matched_by_url: print(f" Match card {i} by URL: {product_url}") else: print(f" Match card {i}: '{matched_title}'") driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", card) time.sleep(1) action_buttons = card.find_elements( By.XPATH, ".//button | .//a[contains(@class, 'button') or contains(@class, 'btn') or @role='button']", ) target = None fallback_add_target = None for btn in action_buttons: text = btn.text.strip().upper() if "SHOP NOW" in text: target = btn break if fallback_add_target is None and ("ADD TO BAG" in text or "ADD TO CART" in text): fallback_add_target = btn if not target and fallback_add_target: target = fallback_add_target if not target: product_links = card.find_elements(By.CSS_SELECTOR, "a[href*='/products/']") if card.tag_name.lower() == "a" and "/products/" in (card.get_attribute("href") or ""): product_links = [card] + product_links if not product_links: continue if not robust_click(product_links[0]): continue print(" Opened product page directly") time.sleep(3) return True, "OPEN PRODUCT PAGE" label = target.text.strip().upper() clicked = robust_click(target) if not clicked: continue print(f" Clicked action: {label}") time.sleep(3) return True, label except Exception: continue if product_url and looks_like_product_url(product_url): try: print(f" Collection cards unavailable, opening product URL directly: {product_url}") driver.get(product_url) time.sleep(2.5) accept_cookies_once() return True, "OPEN PRODUCT URL" except Exception: pass return False, "" def add_to_cart_from_popup_or_page(action_label: str, qty_before: int) -> bool: accept_cookies_once() action_up = (action_label or "").upper() if "ADD TO BAG" in action_up or "ADD TO CART" in action_up: deadline = time.time() + 8 while time.time() < deadline: qty_now = get_cart_quantity() if qty_now > qty_before: return True time.sleep(0.6) drawer_selectors = [ "aside[role='dialog']", "[role='dialog']", ".drawer", ".quick-add", ".product-drawer", ".slideout", ] for selector in drawer_selectors: try: wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, selector))) break except TimeoutException: continue # Drawer uses same-page quick add; wait for it to render fully. try: wait.until( EC.any_of( EC.presence_of_element_located( (By.XPATH, "//*[contains(translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), 'choose your shade')]") ), EC.presence_of_element_located( (By.XPATH, "//button[contains(translate(., 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO BAG')]") ), ) ) except TimeoutException: pass try: variant_selectors = driver.find_elements( By.CSS_SELECTOR, "aside select[name='id'], .drawer select[name='id'], " "form[action*='/cart/add'] select, select[name='id'], .single-option-selector", ) for selector in variant_selectors: if selector.is_displayed() and selector.is_enabled(): sel = Select(selector) sel.select_by_index(0) time.sleep(0.5) break except Exception: pass # If there are visible swatches, click first selected/first available to ensure add is enabled. try: swatches = driver.find_elements( By.XPATH, "//*[self::button or self::label or self::a][@role='radio' " "or contains(@class, 'swatch') or contains(@class, 'color') " "or contains(@class, 'variant') or @data-variant-option-value]", ) for swatch in swatches[:10]: if swatch.is_displayed() and robust_click(swatch): time.sleep(0.4) break except Exception: pass add_xpath_candidates = [ "//aside//button[contains(translate(., 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO BAG')]", "//button[contains(translate(., 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO BAG')]", "//button[contains(translate(., 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'), 'ADD TO CART')]", "//*[@name='add']", "//*[contains(@class, 'add-to-cart') and (self::button or self::a)]", ] add_clicked = False deadline = time.time() + 10 while time.time() < deadline and not add_clicked: for xpath in add_xpath_candidates: buttons = driver.find_elements(By.XPATH, xpath) if not buttons: continue for btn in buttons: try: if not btn.is_displayed(): continue if not element_is_enabled(btn): continue if not robust_click(btn): continue except StaleElementReferenceException: continue print(" Clicked ADD TO BAG") add_clicked = True break if add_clicked: break if not add_clicked: time.sleep(0.6) if add_clicked: time.sleep(3) qty_after = get_cart_quantity() return qty_after > qty_before best_add = find_best_visible_add_button() if best_add and robust_click(best_add): print(" Clicked ADD TO BAG") time.sleep(3) qty_after = get_cart_quantity() return qty_after > qty_before # final verification path qty_after = get_cart_quantity() if qty_after > qty_before: print(" Product quantity increased in cart") return True print(" ADD TO BAG button was not clickable") return False def open_checkout_and_get_url() -> str: current_checkout = wait_for_checkout_url(1) if current_checkout: return current_checkout driver.get(CART_URL) time.sleep(2) if not cart_has_item(): return "" try: checkout_button = WebDriverWait(driver, 12).until( EC.element_to_be_clickable((By.XPATH, CHECKOUT_BUTTON_XPATH)) ) if not robust_click(checkout_button): return "" checkout_url = wait_for_checkout_url(20) if checkout_url: return checkout_url driver.get("https://www.laurageller.com/checkout") checkout_url = wait_for_checkout_url(20) return checkout_url except TimeoutException: return "" def ensure_output_columns(frame: pd.DataFrame) -> pd.DataFrame: if "Status" not in frame.columns: frame["Status"] = "" if "Checkout_URL" not in frame.columns: frame["Checkout_URL"] = "" return frame try: df = pd.read_excel(INPUT_FILE) df = ensure_output_columns(df) titles = df["Product_Title"].fillna("").tolist() urls = df["URL"].fillna("").tolist() if "URL" in df.columns else [""] * len(df) for index, title in enumerate(titles): if not title.strip(): continue print(f"\nProcessing ({index + 1}/{len(titles)}): {title}") qty_before = get_cart_quantity() product_url = urls[index] if index < len(urls) else "" product_opened, action_label = find_and_open_product(title, product_url) if not product_opened: df.loc[index, "Status"] = "Product not found" print(" Product not found on best-seller pages") continue added = add_to_cart_from_popup_or_page(action_label, qty_before) if not added: df.loc[index, "Status"] = "Add to cart failed" continue df.loc[index, "Status"] = "Added to bag" output_df = df.drop(columns=["Status", "Checkout_URL"], errors="ignore") output_df.to_excel(OUTPUT_FILE, index=False) print(f"\nDone. Updated file saved: {OUTPUT_FILE}") finally: driver.quit()