Python 爬虫笔记（由站内到站外爬虫）

#! /usr/bin/env python#coding=utf-8import urllib2frombs4 importBeautifulSoupimportreimport datetimeimport randompages=set()random.seed(datetime.datetime.now())#Retrieves a list of all In

张章章Sam

1235人浏览 · 2016-09-28 12:04:07

张章章Sam · 2016-09-28 12:04:07 发布

#! /usr/bin/env python
#coding=utf-8

import urllib2
from    bs4 import  BeautifulSoup
import  re
import datetime
import random

pages=set()
random.seed(datetime.datetime.now())
#Retrieves a list of all Internal links found on a page
def getInternalLinks(bsObj, includeUrl):
                internalLinks   =   []
                #Finds  all links   that    begin   with    a   "/"
                for link    in  bsObj.findAll("a",  href=re.compile("^(/|.*"+includeUrl+")")):
                                if  link.attrs['href']  is  not None:
                                                if  link.attrs['href']  not in  internalLinks:
                                                                internalLinks.append(link.attrs['href'])
                return  internalLinks
#Retrieves a list of all external links found on a page
def getExternalLinks(bsObj, excludeUrl):
                externalLinks   =   []
                #Finds  all links   that    start   with    "http"  or  "www"   that    do
                #not    contain the current URL
                for link    in  bsObj.findAll("a",  
                                                            href=re.compile("^(http|www)((?!"+excludeUrl+").)*$")):
                                if  link.attrs['href']  is  not None:
                                                if  link.attrs['href']  not in  externalLinks:
                                                                externalLinks.append(link.attrs['href'])
                return  externalLinks

def splitAddress(address):
                addressParts    =   address.replace("http://",  "").split("/")
                return  addressParts

def getRandomExternalLink(startingPage):
                html=   urllib2.urlopen(startingPage)
                bsObj   =   BeautifulSoup(html)
                externalLinks   =   getExternalLinks(bsObj, splitAddress(startingPage)[0])
                if  len(externalLinks)  ==  0:
                                internalLinks   =   getInternalLinks(startingPage)
                                return  internalLinks[random.randint(0, len(internalLinks)-1)]
                else:
                                return  externalLinks[random.randint(0, len(externalLinks)-1)]

def followExternalOnly(startingSite):
                externalLink=getRandomExternalLink("http://oreilly.com")
                print("Random   external    link    is: "+externalLink)
                followExternalOnly(externalLink)

followExternalOnly("http://oreilly.com")