Web scraping
with Python

Philo van Kemenade - @phivk

General Assembly London

Homework

This workshop

these slides

bit.ly/scrapingwithpython

Data on the web

Web Scraping

Pseudo code

1. Read page HTML

from urllib.request import urlopen
url = 'http://isitweekendyet.com/'
pageSource = urlopen(url).read()

2. Parse from Page to soup

from bs4 import BeautifulSoup
weekendSoup = BeautifulSoup(pageSource, "html.parser")

   

>>> weekendSoup.title
 <title>Is it weekend yet?</title>                     

3. Extract what we’re looking for

Mini Project

Questions?

Useful Resources

Useful Modules etc

 Thank you 

Navigating to the weekend yet answer

>>> from urllib.request import urlopen
>>> from bs4 import BeautifulSoup
>>> url = "http://isitweekendyet.com/"
>>> source = urlopen(url).read()
>>> soup = BeautifulSoup(source)

>>> soup.body.div.string
'\nNO\n'

# an alternative:
>>> list(soup.body.stripped_strings)[0]
'NO' 

Book Store

'''A simple script that scrapes info about Books'''

# import modules
from urllib.request import urlopen
from bs4 import BeautifulSoup

# open webpage
urlBooks = "http://books.toscrape.com/"
pageSourceBooks = urlopen(urlBooks).read()

# parse HTML into Beautiful Soup
booksSoup = BeautifulSoup(pageSourceBooks, "html.parser")

# extract data from parsed soup
booksOneStar = booksSoup.find_all('p', class_="star-rating One")
oneStarCount = len(booksOneStar)
print(oneStarCount)

#########
# Bonus #
#########

# simple approach

books1StarCount = len(booksSoup.find_all('p', class_="star-rating One"))
books2StarCount = len(booksSoup.find_all('p', class_="star-rating Two"))
books3StarCount = len(booksSoup.find_all('p', class_="star-rating Three"))
books4StarCount = len(booksSoup.find_all('p', class_="star-rating Four"))
books5StarCount = len(booksSoup.find_all('p', class_="star-rating Five"))

print("1 star: ", books1StarCount)
print("2 star: ", books2StarCount)
print("3 star: ", books3StarCount)
print("4 star: ", books4StarCount)
print("5 star: ", books5StarCount)

# more elegant approach

def getStarCount(booksSoup, starClass):
  booksWithStarClass = booksSoup.find_all('p', class_="star-rating "+starClass)
  starCount = len(booksWithStarClass)
  return starCount

starClasses = ["One", "Two", "Three", "Four", "Five"]

starCounts = {}
for starClass in starClasses:
  starCount = getStarCount(booksSoup, starClass)
  starCounts[starClass] = starCount

print(starCounts)

THE Bitcoin converter

 '''A simple script that converts BTC to GBP based on live rate'''

# import modules
from urllib.request import urlopen
from bs4 import BeautifulSoup

# open webpage
url = "https://exchangerate.guru/btc/"
pageSource = urlopen(url).read()

# parse HTML into Beautiful Soup
btcSoup = BeautifulSoup(pageSource, "html.parser")

# extract data from parsed soup

gbpRateTag = btcSoup.find('a', href='/btc/gbp/1/')
gbpRate = float(gbpRateTag.string)

# get input from user

noBitcoinString = input('How many Bitcoin have you got? ')
noBitcoin = float(noBitcoinString)

# calculate and print answer
noGBP = noBitcoin * gbpRate

print("WOW! you have " + str(noGBP) + "£!!!")

A LiteCoin converter

'''
LiteCoin converter that tells us how much your LiteCoins are worth in EURO
NB: expects python3
'''

# import modules
from urllib.request import urlopen
from bs4 import BeautifulSoup

def clean_up_rate(rateString):
    ''''Clean up raw rateString to form rate'''
    rateStringStripped = rateString.strip()
    rateNumber = rateStringStripped[1:10]
    return float(rateNumber)

# open webpage
url = "http://litecoinexchangerate.org/c/EUR"
pageSource = urlopen(url).read()

# turn html into beautiful soup
liteCoinSoup = BeautifulSoup(pageSource, "html.parser")

# extract info from soup
rateString = liteCoinSoup.find('b').string

# clean up data
rate = clean_up_rate(rateString)

# get user input
litecoinsString = input("How many litecoins have you got?\n>>> ")
litecoins = float(litecoinsString)

# print output
EURO = rate * litecoins
print("You have", round(EURO,2), "EURO!")

A bitcoin converter

'''
Bitcoin converter that tells us how much your bitcoins are worth in GBP
NB: expects python3
'''

# import modules
from urllib.request import urlopen
from bs4 import BeautifulSoup

def clean_up_rate(rateString):
    ''''Clean up raw rateString to form rate'''
    rateNumber = rateString[1:10]
    return float(rateNumber)

def main():
    '''Our main function that gets called when we run the program'''
    # open webpage
    url = "http://bitcoinexchangerate.org/c/GBP/1"
    webpage = urlopen(url).read()

    # turn html into beautiful soup
    bitcoinSoup = BeautifulSoup(webpage, "html.parser")

    # extract info from soup
    rateString = bitcoinSoup.find('b').string.strip()

    # clean up data
    rate = clean_up_rate(rateString)

    # get user input
    bitcoinsString = input("How many bitcoins have you got?\n>>> ")
    bitcoins = float(bitcoinsString)

    # print output
    GBP = rate * bitcoins
    print("You have", round(GBP,2), "GBP!")

# this kicks off our program & lets us both run and import the program
if __name__ == '__main__':
    main()