1
import asyncio
2
from typing import Any
3
import urllib.parse
4
import sys
5
from playwright.async_api import async_playwright
6
import difflib
7
from typing_extensions import TypedDict
8
import os
9

10
def search_query(name_of_hotel: str):
11
    encoded_query = urllib.parse.quote(name_of_hotel)
12
    maps_url = f"https://www.google.com/maps/search/?api=1&query={encoded_query}"
13
    return maps_url
14

15
def get_similarity(search_term: str, actual_name: str) -> float:
16
    search_clean = search_term.lower().strip()
17
    actual_clean = actual_name.lower().strip()
18
    similarity = difflib.SequenceMatcher(None, search_clean, actual_clean).ratio()
19
    return round(similarity, 2)
20

21
async def Login_and_Save_Google_State(browser: Any) -> bool:
22
    context = await browser.new_context()
23
    page = await context.new_page()
24
    await page.goto("https://accounts.google.com/")
25

26
    print("Please log in to your Google account in the browser.", file=sys.stderr)
27
    print("Once you are fully logged in, come back here and press ENTER.", file=sys.stderr)
28

29
    await page.wait_for_timeout(1000 * 60)
30
    await context.storage_state(path="google_auth.json")
31
    print("Login state saved to google_auth.json!", file=sys.stderr)
32

33
    await page.close()
34
    await context.close()
35
    return True
36

37
# --- THE MAIN SCRAPING FUNCTION ---
38
async def search_maps_tab(context: Any, name_of_hotel: str) -> dict[str, float | int]:
39
    chosen_rating: float = 0.0
40
    num_of_reviews: int = 0
41

42
    # Open a new tab in the shared browser context
43
    page = await context.new_page()
44

45
    try:
46
        print(f"Starting the Scraping Process for {name_of_hotel}", file=sys.stderr)
47
        maps_url: str = search_query(name_of_hotel)
48
        print(f"🏃 Navigating directly to: {maps_url}", file=sys.stderr)
49

50
        try:
51
            await page.goto(maps_url, wait_until="networkidle", timeout=1000)
52
        except Exception:
53
            pass # Network idle timeout reached
54

55
        isResultsPageVisible: bool = False
56
        try:
57
            print("🔎 checking if Results Heading is Visible", file=sys.stderr)
58
            resultsHeading = page.get_by_role("heading", name="Results", exact=True)
59
            await resultsHeading.first.wait_for(state="visible", timeout=3000)
60
            isResultsPageVisible = await resultsHeading.is_visible()
61
        except Exception:
62
            try:
63
                print("🔎 checking if You're at the end of the list", file=sys.stderr)
64
                endOfListSpan = page.get_by_text("You've reached the end of the list.", exact=True)
65
                await endOfListSpan.wait_for(state="visible", timeout=3000)
66
                isResultsPageVisible = await endOfListSpan.is_visible()
67
            except Exception:
68
                isResultsPageVisible = False
69

70
        # Nested direct scrape function
71
        async def direct_google_map_page_scrape(page: Any, chosen_rating: float = 0.0, num_of_reviews: int = 0) -> dict[str, float | int]:
72
            try:
73
                await page.locator('button[aria-label^="Photo of "]').first.wait_for(state="visible", timeout=10000)
74
            except Exception:
75
                await page.wait_for_timeout(1000)
76

77
            max_attempts = 5
78
            attempt = 0
79
            while chosen_rating == 0.0 and attempt < max_attempts:
80
                all_ratings = await page.locator('div[tabindex="-1"] div.fontBodyMedium span[aria-hidden="true"]').all_inner_texts()
81
                span_texts = [text.strip() for text in all_ratings if text.isascii() and text.strip()]
82

83
                for text in span_texts:
84
                    try:
85
                        chosen_rating = float(text)
86
                        break
87
                    except ValueError:
88
                        pass
89

90
                attempt += 1
91
                await page.wait_for_timeout(100)
92

93
            await page.wait_for_timeout(1000)
94

95
            if chosen_rating != 0.0:
96
                print(f"✅ Ratings Found! {chosen_rating} Moving on to reviews...", file=sys.stderr)
97
                await page.wait_for_timeout(1000)
98
                reviews_locator = page.locator('div[tabindex="-1"] span[aria-label*="review"]').first
99

100
                try:
101
                    await reviews_locator.wait_for(state="attached", timeout=5000)
102
                    aria_label = await reviews_locator.get_attribute("aria-label")
103
                    if aria_label:
104
                        num_of_reviews_str = ''.join(filter(str.isdigit, aria_label))
105
                        if num_of_reviews_str:
106
                            num_of_reviews = int(num_of_reviews_str)
107
                            print(f"✅ Number of reviews found: {num_of_reviews}", file=sys.stderr)
108
                except Exception:
109
                    print("❌ No reviews element found (Timeout).", file=sys.stderr)
110

111
            await page.wait_for_timeout(1000)
112
            return {
113
                "rating": chosen_rating,
114
                "num_of_reviews": num_of_reviews,
115
            }
116

117
        if isResultsPageVisible:
118
            print("📍 You're in the results page", file=sys.stderr)
119
            links = page.locator("a[aria-label]")
120

121
            class HotelResult(TypedDict, closed=True):
122
                hotel_name: str
123
                similarity_score: float
124
                rating: float
125
                href_link: str
126

127
            listThing: list[HotelResult] = []
128

129
            js_code = """
130
            elements => elements.map(e => {
131
                let rating = 0.0;
132
                const ratingSpan = e.querySelector('span[aria-label*="stars"], span[aria-label*="star"]');
133
                if (ratingSpan) {
134
                    const aria = ratingSpan.getAttribute('aria-label');
135
                    const match = aria.match(/[\\d\\.]+/);
136
                    if (match) {
137
                        rating = parseFloat(match[0]);
138
                    }
139
                }
140
                return {
141
                    name: e.getAttribute('aria-label'),
142
                    rating: rating,
143
                    href: e.getAttribute('href') || ''
144
                };
145
            })
146
            """
147
            all_hotel_data = await links.evaluate_all(js_code)
148
            for data in all_hotel_data:
149
                hotel_name_res = data['name']
150
                href_link = data['href']
151
                extracted_rating = data['rating']
152
                similarity = get_similarity(name_of_hotel, hotel_name_res)
153

154
                listThing.append(HotelResult(
155
                    hotel_name=hotel_name_res,
156
                    similarity_score=similarity,
157
                    rating=extracted_rating,
158
                    href_link=href_link
159
                ))
160

161
            listThing.sort(key=lambda x: (x['similarity_score'], x['rating']), reverse=True)
162

163
            if listThing:
164
                print(f"✅ Best Match: {listThing[0]['hotel_name']}", file=sys.stderr)
165
                await page.goto(f"{listThing[0]['href_link']}")
166
            await page.wait_for_timeout(3000)
167

168
            responseDirectGoogeleMapScrape = await direct_google_map_page_scrape(page)
169
            chosen_rating = responseDirectGoogeleMapScrape["rating"]
170
            num_of_reviews = int(responseDirectGoogeleMapScrape["num_of_reviews"])
171
        else:
172
            print("📍 You're in the direct Google Map Page", file=sys.stderr)
173
            responseDirectGoogeleMapScrape = await direct_google_map_page_scrape(page)
174
            chosen_rating = responseDirectGoogeleMapScrape["rating"]
175
            num_of_reviews = int(responseDirectGoogeleMapScrape["num_of_reviews"])
176

177
    except Exception as e:
178
        print(f"❌ Error during scraping {name_of_hotel}: {e}", file=sys.stderr)
179
    finally:
180
        # ALWAYS CLOSE THE TAB
181
        await page.close()
182

183
    return {
184
        "rating": chosen_rating,
185
        "num_of_reviews": num_of_reviews,
186
    }

1
import threading
2
import random
3
from dotenv import load_dotenv
4
from typing import Any, cast
5

6
# Limit Playwright to 10 active tabs at a time
7
MAX_TABS = 20
8
semaphore = asyncio.Semaphore(MAX_TABS)
9

10
cached_list: list[dict[str, Any]] = []
11
total_hotels = len(filteredDF) # Assuming filtered_df is your loaded dataframe
12

13
async def process_hotel(context: Any, index: int, row_series: Any):
14
    async with semaphore:
15
        load_dotenv(override=True)
16
        stop_threads_flag = os.getenv("STOP_PROGRAM_THREADS_NUMBER_TWO", "false").lower() == "true"
17

18
        if stop_threads_flag:
19
            return
20
        name_of_hotel = str(row_series['name_of_hotel'])
21
        # Add a tiny async sleep to stagger the tabs opening
22
        await asyncio.sleep(random.uniform(0.1, 1.5))
23

24
        try:
25
            # Call the function we defined in Cell 1!
26
            # We use asyncio.wait_for as our new "Kill Switch" (45 seconds)
27
            result = await asyncio.wait_for(
28
                search_maps_tab(context, name_of_hotel),
29
                timeout=45.0
30
            )
31

32
            cached_list.append({
33
                "index": index,
34
                "rating": result.get('rating'),
35
                "num_of_reviews": result.get('num_of_reviews')
36
            })
37
            print(f"🎯 Saved [{len(cached_list)}/{total_hotels}]: {name_of_hotel} -> {result}")
38

39
        except asyncio.TimeoutError:
40
            print(f"💀 KILLED [{len(cached_list)}]: Tab hung for over 45 seconds --- {name_of_hotel}")
41
        except Exception as e:
42
            print(f"❌ Failed [{len(cached_list)}]: {e} --- {name_of_hotel}")
43

44
async def run_scraper():
45
    print("Launching single browser instance...")
46
    async with async_playwright() as p:
47
        browser = await p.chromium.launch(
48
            headless=False, # Set to False if you want to watch the tabs!
49
            channel="chrome",
50
            args=["--disable-blink-features=AutomationControlled"]
51
        )
52

53
        # Load the cookies ONCE for all tabs to share
54
        if os.path.exists("google_auth.json"):
55
            context = await browser.new_context(
56
                storage_state="google_auth.json",
57
                viewport={"width": 1920, "height": 1080},
58
                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
59
            )
60
        else:
61
            context = await browser.new_context()
62

63
        # Create a task for every row
64
        tasks: list[Any] = []
65
        for index, row in filteredDF.iterrows():
66
            tasks.append(process_hotel(context, cast(int, index), row))
67

68
        print(f"Firing {len(tasks)} concurrent tab tasks (Max {MAX_TABS} at a time)...")
69
        await asyncio.gather(*tasks)
70

71
        await context.close()
72
        await browser.close()
73

74
# --- THE JUPYTER THREADING WRAPPER ---
75
def start_scraping_in_background():
76
    if sys.platform == 'win32':
77
        asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
78
    asyncio.run(run_scraper())
79

80
# Execute the thread
81
print("Spinning up Playwright...")
82
scraper_thread = threading.Thread(target=start_scraping_in_background)
83
scraper_thread.start()
84
scraper_thread.join()
85

86
print("All tabs finished. Merging data back into the DataFrame...")
87
for item in cached_list:
88
    row_idx: int  = cast(int, item["index"])
89
    inputDF.at[row_idx, 'rating'] = item["rating"]
90
    inputDF.at[row_idx, 'num_of_reviews'] = item["num_of_reviews"]
91

92
output_filename = "output-data/hotel_data_fully_scraped.xlsx"
93
os.makedirs("output-data", exist_ok=True)
94
inputDF.to_excel(output_filename, index=False) # type: ignore
95
print(f"🎉 Scraping complete! Data saved to {output_filename}")