自學python爬蟲(六)PyQuery使用
阿新 • • 發佈:2018-11-06
一、概念
PyQuery庫也是很強大的網頁解析庫,適合熟悉Jquery的人使用,因為PyQuery和Jquery的API幾乎一模一樣。
二、詳細講解
1、初始化
1.1字串初始化
# 字串初始化
html = """
<div>
<ul>
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div >
"""
from pyquery import PyQuery as pq
doc = pq(html)
# CSS選擇器
print(doc('li'))
1.2URL初始化
# URL初始化
from pyquery import PyQuery as pq
doc = pq(url='http://www.baidu.com')
print(doc('head'))
1.3檔案初始化
# 檔案初始化
from pyquery import PyQuery as pq
doc = pq(filename='demo.html')
print(doc('li'))
2、基本的CSS選擇器
# 基本CSS選擇器
html = """
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
# container 、list、li之間只要有層級關係就好,不一定非是直接得父子關係
print(doc('#container .list li'))
print(doc('#container .list a'))
3、查詢元素
3.1子元素
html = """
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc(' .list')
print(type(items))
print(items)
lis = items.find('li')
print(type(lis))
print(lis)
lis = items.children()
print(type(lis))
print(lis)
lis = items.children(' .active')
print(lis)
3.2父元素
# 父元素
html = """
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc(' .list')
container = items.parent()
print(type(container))
print(container)
# 父元素
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
items = doc(' .list')
parents = items.parents()
print(type(parents))
print(parents)
parent = items.parents('.wrap')
print(parent)
3.3兄弟元素
# 兄弟元素
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
# 兩個選擇同時滿足
li = doc(' .list .item-0.active')
print(li.siblings())
print(li.siblings(' .active'))
4、遍歷
4.1獲取單個元素
# 獲取單個元素
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc(' .item-0.active')
print(li)
4.2獲取多個元素
# 獲取多個元素
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="#"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
list = doc('li').items()
print(type(list))
for li in list:
print(li)
5、獲取資訊
5.1獲取屬性
# 獲取屬性
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc(' .item-0.active a')
print(a)
print(a.attr('href'))
print(a.attr.href)
5.2獲取文字
# 獲取文字
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc(' .item-0.active a')
print(a)
print(a.text())
5.3獲取HTML
# 獲取HTML
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
a = doc(' .item-0.active')
print(a)
print(a.html())
6、DOM操作
6.1 addClass removeClass
# addClass removeClass
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc(' .item-0.active')
print(li)
li.removeClass('active')
print(li)
li.addClass('active')
print(li)
6.2 attr css
# attr css
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
li = doc(' .item-0.active')
print(li)
# 增加一個屬性name,如果name屬性存在,則把值設為link
li.attr('name','link')
print(li)
# 設定css樣式
li.css('font-size','13px')
print(li)
6.3 remove
# remove
html="""
<div class="wrap">
Hello world
<p>this is a paragraph</p>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
wrap = doc(' .wrap')
print(wrap.text())
wrap.find('p').remove()
print(wrap.text())
7、偽類選擇器
html = """
<div class="wrap">
<div id="container">
<ul class="list">
<li class="item-0">first item</li>
<li class="item-1"><a href="#">second item</a></li>
<li class="item-0 active"><a href="12324.html"><span class="bold">third item</span></a></li>
<li class="item-1 active"><a href="#">fourth item</a></li>
<li class="item-0"><a href="#">fifth item</a></li>
</ul>
</div>
</div>
"""
from pyquery import PyQuery as pq
doc = pq(html)
# first child
li = doc('li:first-child')
print(li)
# last child
li = doc('li:last-child')
print(li)
# second,2可以替換掉
li = doc('li:nth-child(2)')
print(li)
#great than
li = doc('li:gt(2)')
print(li)
# 2n,偶數
li = doc('li:nth-child(2n)')
print(li)
# 包含 second字眼的li
li = doc('li:contains(second)')
print(li)