bs4 Automatically Installed

We are happy to announce that beautifulsoup4 is now automatically installed on LoadForge workers. This allows you to parse html pages and create test scripts which automatically find your static content, urls and more.

from bs4 import BeautifulSoup
from locust import HttpUser, task, between

class WebsiteUser(HttpUser):
    wait_time = between(1, 2.5)
    
    def on_start(self):
        self.crawl("/")
    
    def crawl(self, path):
        response = self.client.get(path, name="[Page] "+path)
        if response.status_code == 200:
            soup = BeautifulSoup(response.text, "html.parser")
            
            # Discover linked pages
            links = [a["href"] for a in soup.find_all("a", href=True)]
            for link in links:
                self.crawl(link)
            
            # Load static content (img, css, js)
            resources = [(link["href"], "CSS") for link in soup.find_all("link", href=True)]
            resources += [(script["src"], "JS") for script in soup.find_all("script", src=True)]
            resources += [(img["src"], "IMG") for img in soup.find_all("img", src=True)]

            for resource, res_type in resources:
                self.client.get(resource, name=f"[{res_type}] {resource}")
    
    @task(1)
    def load_homepage(self):
        self.client.get("/")

You can begin using bs4 immediately.