Untitled

#%%
import os
import requests
from bs4 import BeautifulSoup
import re

search_term = 'Dark-eyed+Junco'  # change this to your desired search term

# build the search URL
url = f'https://www.allaboutbirds.org/news/search/?q={search_term}'

# send a GET request to the search URL
response = requests.get(url, stream=True)

# parse the HTML content of the response using BeautifulSoup
soup = BeautifulSoup(response.content, 'html.parser')
print(soup)
# find all the search result items on the page
search_results = soup.find_all('div', class_='species-search-results')

# print the titles and URLs of the search result items
for result in search_results:
    url = result.find('a', class_='article-item-link').text.strip()
    title = url.split('/')[-2]
    print(url, title)
    # url = result.find('a', class_='search-result__link')['href']
    # print(f'Title: {title}\nURL: {url}\n')
# %%