Puppeteer กับ Selenium: จะเลือกอะไรดีสำหรับการขูดเว็บ?

const puppeteer = require('puppeteer'); 

async function main() { 
    // Launch a headless browser instance 
    const browser = await puppeteer.launch({ headless: true });

    // Create a new page object 
    const page = await browser.newPage();

    // Navigate to the target URL and wait until the loading finishes
    await page.goto('https://danube-webshop.herokuapp.com/', { waitUntil: 'networkidle2' });

    // Wait for the left-side bar to load
    await page.waitForSelector('ul.sidebar-list');

    // Click on the first element and wait for the navigation to finish
    await Promise.all([
        page.waitForNavigation(),
        page.click("ul[class='sidebar-list'] > li > a"),
    ]);

    // Wait for the book previews to load
    await page.waitForSelector("li[class='preview']");

    // Extract the book previews
    const books = await page.evaluateHandle(
        () => [...document.querySelectorAll("li[class='preview']")]
    );

    // Extract the relevant data using page.evaluate
    const processed_data = await page.evaluate(elements => {
        let data = [];
        elements.forEach(element => {
            let title = element.querySelector("div.preview-title").innerHTML;
            let author = element.querySelector("div.preview-author").innerHTML;
            let rating = element.querySelector("div.preview-details > p.preview-rating").innerHTML;
            let price = element.querySelector("div.preview-details > p.preview-price").innerHTML;

            let result = { title, author, rating, price };
            data.push(result);
        });
        return data;
    }, books);

    // Print out the extracted data
    console.log(processed_data);

    // Close the page and browser respectively
    await page.close();
    await browser.close();
}

// Run the main function to scrape the data
main();

[
    {
        title: 'Does the Sun Also Rise?',
        author: 'Ernst Doubtingway',
        rating: '★★★★☆',
        price: '$9.95'
    },
    {
        title: 'The Insiders',
        author: 'E. S. Hilton',
        rating: '★★★★☆',
        price: '$9.95'
    },
    {
        title: 'A Citrussy Clock',
        author: 'Bethany Urges',
        rating: '★★★★★',
        price: '$9.95'
    }
]

const puppeteer = require('puppeteer');

async function generatePDF() {
    // Launch a headless browser instance
    const browser = await puppeteer.launch({ headless: true });

    // Create a new page object
    const page = await browser.newPage();

    // Navigate to the target URL
    await page.goto('https://example.com', { waitUntil: 'networkidle2' });

    // Generate a PDF from the web page
    await page.pdf({
        path: 'example.pdf', // Output file path
        format: 'A4',        // Paper format
        printBackground: true, // Include background graphics
    });

    // Close the page and browser respectively
    await page.close();
    await browser.close();
}

// Run the function to generate the PDF
generatePDF();

const puppeteer = require('puppeteer');

async function generatePDF() {
    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    await page.goto('https://example.com', { waitUntil: 'networkidle2' });

    await page.pdf({
        path: 'example.pdf',
        format: 'A4',
        printBackground: true,
        landscape: true,
        margin: {
            top: '20px',
            right: '20px',
            bottom: '20px',
            left: '20px',
        },
    });

    await page.close();
    await browser.close();
}

generatePDF();

import time
from selenium import webdriver
from selenium.webdriver.common.by import By

options = webdriver.ChromeOptions()
options.add_argument("--headless")

driver = webdriver.Chrome(options=options)

time.sleep(1)
crime_n_thrillers = driver.find_element(By.CSS_SELECTOR, "ul[class='sidebar-list'] > li")
crime_n_thrillers.click()
time.sleep(1)
books = driver.find_elements(By.CSS_SELECTOR, "div.shop-content li.preview")

time.sleep(1)
crime_n_thrillers = driver.find_element(By.CSS_SELECTOR, "ul[class='sidebar-list'] > li")
crime_n_thrillers.click()
time.sleep(1)
books = driver.find_elements(By.CSS_SELECTOR, "div.shop-content li.preview")

def extract(element):
    title = element.find_element(By.CSS_SELECTOR, "div.preview-title").text
    author = element.find_element(By.CSS_SELECTOR, "div.preview-author").text
    rating = element.find_element(By.CSS_SELECTOR, "div.preview-details p.preview-rating").text
    price = element.find_element(By.CSS_SELECTOR, "div.preview-details p.preview-price").text
    return {"title": title, "author": author, "rating": rating, "price": price}

extracted_data = []
for element in books:
    data = extract(element)
    extracted_data.append(data)

print(extracted_data)
driver.quit()

[
    {'title': 'Does the Sun Also Rise?', 'author': 'Ernst Doubtingway', 'rating': '★★★★☆', 'price': '$9.95'},
    {'title': 'The Insiders', 'author': 'E. S. Hilton', 'rating': '★★★★☆', 'price': '$9.95'},
    {'title': 'A Citrussy Clock', 'author': 'Bethany Urges', 'rating': '★★★★★', 'price': '$9.95'}
]

from selenium import webdriver

options = webdriver.ChromeOptions()
options.add_argument("--headless")

driver = webdriver.Chrome(options=options)

url = "https://example.com"
driver.get(url)

driver.save_screenshot("example_screenshot.png")

driver.quit()

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

def extract(element):
    title = element.find_element(By.CSS_SELECTOR, "div.preview-title").text
    author = element.find_element(By.CSS_SELECTOR, "div.preview-author").text
    rating = element.find_element(By.CSS_SELECTOR, "div.preview-details p.preview-rating").text
    price = element.find_element(By.CSS_SELECTOR, "div.preview-details p.preview-price").text
    return {"title": title, "author": author, "rating": rating, "price": price}

# Start the timer
start_time = time.time()

options = webdriver.ChromeOptions()
options.add_argument("--headless")

# Create a new instance of the Chrome driver
driver = webdriver.Chrome(options=options)

url = "https://danube-webshop.herokuapp.com/"
driver.get(url)

# Click on the Crime & Thrillers category
time.sleep(1)
crime_n_thrillers = driver.find_element(By.CSS_SELECTOR, "ul[class='sidebar-list'] > li")
crime_n_thrillers.click()
time.sleep(1)

# Extract the book previews
books = driver.find_elements(By.CSS_SELECTOR, "div.shop-content li.preview")

extracted_data = []
for element in books:
    data = extract(element)
    extracted_data.append(data)

print(extracted_data)

# End the timer
end_time = time.time()
print(f"The whole script took: {end_time - start_time:.4f} seconds")

driver.quit()

const puppeteer = require('puppeteer');

async function main() {
    const start = Date.now();

    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    await page.goto('https://danube-webshop.herokuapp.com/', { waitUntil: 'networkidle2' });

    await page.waitForSelector('ul.sidebar-list');

    await Promise.all([
        page.waitForNavigation(),
        page.click("ul[class='sidebar-list'] > li > a"),
    ]);

    await page.waitForSelector("li[class='preview']");
    const books = await page.evaluateHandle(
        () => [...document.querySelectorAll("li[class='preview']")]
    );

    const processed_data = await page.evaluate(elements => {
        let data = [];
        elements.forEach(element => {
            let title = element.querySelector("div.preview-title").innerHTML;
            let author = element.querySelector("div.preview-author").innerHTML;
            let rating = element.querySelector("div.preview-details > p.preview-rating").innerHTML;
            let price = element.querySelector("div.preview-details > p.preview-price").innerHTML;

            let result = { title, author, rating, price };
            data.push(result);
        });
        return data;
    }, books);

    console.log(processed_data);
    await page.close();
    await browser.close();

    const end = Date.now();
    console.log(`Execution time: ${(end - start) / 1000} seconds`);
}

main();

const puppeteer = require('puppeteer');

async function main() {
    const proxyServer = 'http://your-proxy-server:port';
    
    const browser = await puppeteer.launch({
        headless: true,
        args: [`--proxy-server=${proxyServer}`]
    });

    const page = await browser.newPage();
    await page.goto('https://example.com', { waitUntil: 'networkidle2' });

    // Perform your web scraping tasks here

    await browser.close();
}

main();

from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType

proxy = Proxy()
proxy.proxy_type = ProxyType.MANUAL
proxy.http_proxy = "your-proxy-server:port"
proxy.ssl_proxy = "your-proxy-server:port"

capabilities = webdriver.DesiredCapabilities.CHROME
proxy.add_to_capabilities(capabilities)

options = webdriver.ChromeOptions()
options.add_argument("--headless")

driver = webdriver.Chrome(desired_capabilities=capabilities, options=options)
driver.get("https://example.com")

# Perform your web scraping tasks here

driver.quit()