🕯️ Magic Note
Beautiful Soup is named after the Lewis Carroll poem “Beautiful Soup” from Alice in Wonderland. The library was created to be “beautiful” to use, turning messy HTML into a well-structured soup that is easy to digest.
Bash
# Install Beautiful Soup
pip install beautifulsoup4
# Optional: Install lxml parser (faster)
pip install lxml
# Optional: Install html5lib (most lenient)
pip install html5lib
Python
from bs4 import BeautifulSoup
# From an HTML string
html_string = “<html><body><h1>Hello</h1></body></html>”
soup = BeautifulSoup(html_string, “html.parser”)
# From a file
with open(“index.html”, “r”, encoding=”utf-8″) as f:
soup = BeautifulSoup(f, “html.parser”)
# From a URL (after downloading with requests)
import requests
response = requests.get(“https://example.com”)
soup = BeautifulSoup(response.text, “html.parser”)
# Using different parsers
soup = BeautifulSoup(html_string, “html.parser”) # Built-in, good enough
soup = BeautifulSoup(html_string, “lxml”) # Faster, needs lxml
soup = BeautifulSoup(html_string, “html5lib”) # Most lenient, needs html5lib
Python
from bs4 import BeautifulSoup
html = “””
<html>
<body>
<h1 id=”main-title” class=”title”>Welcome to My Site</h1>
<p>This is the first paragraph.</p>
<p>This is the second paragraph.</p>
<div>
<a href=”https://example.com”>Example Link</a>
</div>
</body>
</html>
“””
soup = BeautifulSoup(html, “html.parser”)
# Access by tag name (returns the first matching tag)
h1 = soup.h1
print(f”h1 text: {h1.text}”)
print(f”h1 id: {h1.get(‘id’)}”)
print(f”h1 class: {h1.get(‘class’)}”)
# Multiple tags with the same name (returns first only)
first_p = soup.p
print(f”First p text: {first_p.text}”)
# Use find_all() to get all matching tags
all_p = soup.find_all(“p”)
for i, p in enumerate(all_p):
print(f”p[{i}]: {p.text}”)
Python
from bs4 import BeautifulSoup
html = “””
<div id=”container” class=”main”>
<ul>
<li class=”item” data-id=”1″>Item 1</li>
<li class=”item” data-id=”2″>Item 2</li>
<li class=”item special” data-id=”3″>Item 3 (special)</li>
</ul>
<p class=”description”>Some description here.</p>
<p>Another paragraph without class.</p>
</div>
“””
soup = BeautifulSoup(html, “html.parser”)
# find() – returns first matching element
first_li = soup.find(“li”)
print(f”First li: {first_li.text}”)
# find_all() – returns list of all matching elements
all_li = soup.find_all(“li”)
print(f”All li: {len(all_li)} found”)
# Find by class
description = soup.find(class_=”description”) # Note: class_ (underscore)
print(f”By class: {description.text}”)
# Find all elements with a specific class
items = soup.find_all(class_=”item”)
print(f”Items: {[item.text for item in items]}”)
# Find by ID
container = soup.find(id=”container”)
print(f”By ID: {container.get(‘class’)}”)
# Find by attribute
item_with_data = soup.find(attrs={“data-id”: “2”})
print(f”By data-id=2: {item_with_data.text}”)
# Find with multiple conditions
special_item = soup.find(“li”, class_=”special”)
print(f”Special item: {special_item.text}”)
🕯️ Magic Note
The find_all() method has many useful parameters: limit to limit results, recursive=False to search only direct children, and string to search by text content. These make complex searches simple.
Python
from bs4 import BeautifulSoup
html = “””
<div id=”container”>
<ul class=”menu”>
<li><a href=”/home”>Home</a></li>
<li><a href=”/about” class=”active”>About</a></li>
<li><a href=”/contact”>Contact</a></li>
</ul>
<div class=”content”>
<p class=”first”>First paragraph.</p>
<p>Second paragraph.</p>
</div>
</div>
“””
soup = BeautifulSoup(html, “html.parser”)
# Basic CSS selectors
menu_items = soup.select(“ul.menu li”)
print(f”Menu items: {len(menu_items)}”)
# Select by ID (#id)
container = soup.select_one(“#container”)
print(f”Container found: {container is not None}”)
# Select by class (.class)
active_link = soup.select_one(“a.active”)
print(f”Active link: {active_link.get(‘href’)}”)
# Select by attribute
first_para = soup.select_one(“p[class=’first’]”)
print(f”First paragraph: {first_para.text}”)
# Select with multiple conditions
active_menu = soup.select(“ul.menu li a.active”)
if active_menu:
print(f”Active menu: {active_menu[0].get(‘href’)}”)
Python
from bs4 import BeautifulSoup
html = “””
<div class=”container”>
<ul>
<li>First</li>
<li>Second</li>
<li>Third</li>
</ul>
<p>Footer text</p>
</div>
“””
soup = BeautifulSoup(html, “html.parser”)
# Start with the second li
second_li = soup.find_all(“li”)[1] # Second li element
print(f”Current: {second_li.text}”)
# Parent
parent_ul = second_li.parent
print(f”Parent: {parent_ul.name}”)
# Grandparent
grandparent = parent_ul.parent
print(f”Grandparent: {grandparent.get(‘class’)}”)
# Previous sibling
prev_sibling = second_li.previous_sibling
if prev_sibling and prev_sibling.name == “li”:
print(f”Previous sibling: {prev_sibling.text}”)
# Next sibling
next_sibling = second_li.next_sibling
if next_sibling and next_sibling.name == “li”:
print(f”Next sibling: {next_sibling.text}”)
# All siblings
for sibling in second_li.previous_siblings:
if sibling.name == “li”:
print(f”Earlier sibling: {sibling.text}”)
Python
from bs4 import BeautifulSoup
html = “””
<div class=”product” id=”prod-123″>
<h2>Python Book</h2>
<p class=”price”>$29.99</p>
<a href=”/products/123″ data-id=”123″ target=”_blank”>View Details</a>
<div>Additional <span>info</span> here.</div>
</div>
“””
soup = BeautifulSoup(html, “html.parser”)
div = soup.find(“div”, class_=”product”)
# Get text (strips HTML tags)
print(f”.text: {div.text.strip()}”)
print(f”.get_text(): {div.get_text(strip=True)}”)
# Get attribute values
product_id = div.get(“id”)
product_class = div.get(“class”)
print(f”id: {product_id}”)
print(f”class: {product_class}”)
# Get attribute with default value
missing = div.get(“data-missing”, “default”)
print(f”Missing attribute: {missing}”)
# Get child elements
children = list(div.children)
print(f”Direct children: {len(children)}”)
# Get all descendants
descendants = list(div.descendants)
print(f”All descendants: {len(descendants)}”)
Python
from bs4 import BeautifulSoup
html = “””
<ul>
<li>Apple</li>
<li>Banana</li>
<li>Cherry</li>
<li>Date</li>
</ul>
“””
soup = BeautifulSoup(html, “html.parser”)
# Find li containing exact text
banana_li = soup.find(“li”, string=”Banana”)
print(f”Exact match: {banana_li}”)
# Find using function
def contains_an(text):
return text and “an” in text.lower()
matching = soup.find_all(“li”, string=contains_an)
print(f”Contains ‘an’: {[item.text for item in matching]}”)
# Find by text regex
import re
pattern = re.compile(r”a”, re.IGNORECASE)
matching = soup.find_all(“li”, string=pattern)
print(f”Contains ‘a’ (case-insensitive): {[item.text for item in matching]}”)
Python
import requests
from bs4 import BeautifulSoup
import time
import csv
def scrape_quotes(base_url=”http://quotes.toscrape.com”):
“””Scrape quotes and authors from quotes.toscrape.com.”””
session = requests.Session()
session.headers.update({
“User-Agent”: “Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36”
})
quotes_data = []
page = 1
while True:
url = f”{base_url}/page/{page}/” if page > 1 else base_url
print(f”Scraping page {page}…”)
try:
response = session.get(url, timeout=10)
response.raise_for_status()
except requests.exceptions.RequestException as e:
print(f”Error fetching {url}: {e}”)
break
soup = BeautifulSoup(response.text, “html.parser”)
quotes = soup.find_all(“div”, class_=”quote”)
if not quotes:
print(“No more quotes found. Stopping.”)
break
for quote_div in quotes:
text = quote_div.find(“span”, class_=”text”)
author = quote_div.find(“small”, class_=”author”)
tags = quote_div.find_all(“a”, class_=”tag”)
quotes_data.append({
“text”: text.text if text else “N/A”,
“author”: author.text if author else “N/A”,
“tags”: “, “.join(tag.text for tag in tags)
})
# Check if there is a next page
next_button = soup.find(“li”, class_=”next”)
if not next_button:
print(“No next page. Finished scraping.”)
break
page += 1
time.sleep(0.5) # Be polite to the server
print(f”Scraped {len(quotes_data)} quotes”)
# Save to CSV
with open(“quotes.csv”, “w”, newline=””, encoding=”utf-8″) as f:
writer = csv.DictWriter(f, fieldnames=[“text”, “author”, “tags”])
writer.writeheader()
writer.writerows(quotes_data)
return quotes_data
if __name__ == “__main__”:
quotes = scrape_quotes()
for quote in quotes[:3]:
print(f”‘{quote[‘text’]}’ – {quote[‘author’]}”)
🕯️ Magic Note
The site quotes.toscrape.com is another safe practice site for web scraping. It has multiple pages and a clear structure, making it perfect for learning.
Python
# Pattern 1: Safe element extraction
def extract_text(element, selector, default=”N/A”):
“””Safely extract text from a selector within an element.”””
found = element.select_one(selector)
return found.text.strip() if found else default
# Pattern 2: Extract all links
def get_all_links(soup, base_url=””):
links = []
for a in soup.find_all(“a”, href=True):
href = a[“href”]
if base_url and href.startswith(“/”):
href = base_url + href
links.append({
“text”: a.text.strip(),
“href”: href
})
return links
# Pattern 3: Extract table data
def extract_table(table_element):
headers = []
for th in table_element.find_all(“th”):
headers.append(th.text.strip())
rows = []
for tr in table_element.find_all(“tr”):
cells = tr.find_all(“td”)
if cells:
row = [cell.text.strip() for cell in cells]
rows.append(row)
return {“headers”: headers, “rows”: rows}
- Forgetting to handle missing elements (causing AttributeError)
- Using find_all() when find() would suffice (less efficient)
- Not using .strip() on extracted text (capturing whitespace)
- Parsing JSON or XML with BeautifulSoup (use json module for JSON)
- Assuming elements will always have the attributes you expect
- Not handling dynamic JavaScript content (use Selenium for JS-heavy sites)
- What is the difference between find() and find_all()?
- How do you find all elements with a specific class?
- What is the advantage of using CSS selectors with select()?
- How do you extract the text content of an element without HTML tags?
- What is the difference between .text and .get_text()?
- How do you find an element by its ID?
⚡ Whisper
Beautiful Soup turns messy HTML into delicious data. It is patient with broken tags, forgiving with missing attributes, and gentle with malformed documents. You give it a page. It gives you a tree. You search with find_all() and select(). You navigate with parent and children. You extract with .text and .get(). The data emerges. A list of products. A table of prices. A collection of quotes. All ready for your analysis. Beautiful Soup does not scrape—you scrape. Beautiful Soup helps you parse. Learn its methods. Practice on practice sites. Handle missing elements gracefully. Use .strip() for clean text. And remember: Beautiful Soup parses what is there. It cannot run JavaScript. It cannot fill forms. For dynamic sites, you need Selenium (next lesson). But for static HTML, Beautiful Soup is your best friend. Use it well. The web holds endless data. Beautiful Soup helps you collect it.
