AllLinks method

Method that have an issue
 avatar
unknown
python
3 years ago
979 B
3
Indexable
def AllLinks():
    container = []
    for j in range(1900, date.today().year):
        start_from = j
        endfrom = j
        start = 0
        first_page = True
        while True:
            if start == 0:
                url = "https://finder.startupnationcentral.org/startups/search?founded_from_year={}&founded_to_year={}".format(start_from, endfrom,start)
            else:
                url = "https://finder.startupnationcentral.org/startups/search_more?founded_from_year={}&founded_to_year={}&next_page_id={}".format(start_from, endfrom,start)
            print("visiting url is ",url)
            response = requests.get(url)
            soup = BeautifulSoup(response.text,"html.parser")
            urls = ["https://finder.startupnationcentral.org{}".format(i['href']) for i in soup.findAll("a",{"data-report-action" : "Companies"})]
            if len(urls)<1:
                break
            container.extend(urls)
            start+=1
    return container
Editor is loading...