input: Write a function to implement a web crawler in Python that starts from a given URL, extracts all the links from the page using BeautifulSoup, and scrapes the page content. The crawler should continue to follow new links until there are no more pages to visit. The function should return a set of all visited URLs.
  
  
  import requests
  from bs4 import BeautifulSoup
  import re
  
  def crawl(url):
   visited = set()
   queue = [url] 
   
   while queue:
   curr = queue.pop(0)
   visited.add(curr) 
   response = requests.get(curr)
   
   if response.ok:
   soup = BeautifulSoup(response.text, 'html.parser')
  
  # scrape the page here 
  
  # find all the links
   links = soup.find_all('a', href=re.compile('^http')) 
   
   for link in links:
   if link['href'] not in visited:
   queue.append(link['href'])
  
  return visited
  
  assert crawl('http://example.com') == {'http://example.com', 'http://example.com/page1', 'http://example.com/page2'}
  assert crawl('http://test.com') == {'http://test.com', 'http://test.com/subpage'}
  
output: 
  def crawl(url):
   visited = set()
   queue = [url] 
   
   while queue:
   curr = queue.pop(0)
   visited.add(curr) 
   response = requests.get(curr)
   
   if response.ok:
   soup = BeautifulSoup(response.text, 'html.parser')
  
  # scrape the page here 
  
  # find all the links
   links = soup.find_all('a', href=re.compile('^http')) 
   
   for link in links:
   if link['href'] not in visited:
   queue.append(link['href'])
  
  return visited
  