es操作+python連線es+搭建叢集es
阿新 • • 發佈：2022-03-14
1 Elasticsearch的文件增刪查改(CURD)
## 新增文件(即便型別和索引不存在，也能增加)
PUT lqz/_doc/1
{
"name":"顧老二",
"age":30,
"from": "gu",
"desc": "面板黑、武器長、性格直",
"tags": ["黑", "長", "直"]
}
## 更新（兩種都可以）
POST lqz/_doc/1/_update
{
"doc": {
"desc": "面板很黃，武器很長，性格很直",
"tags": ["很黃","很長", "很直"]
}
}
POST lqz/_update/1/
{
"doc": {
"desc": "確實很黃，武器很長，性格很直",
"tags": ["很黃黃","很長", "很直"]
}
}
## 刪除
DELETE lqz/_doc/4
### 簡單查詢
GET lqz/_doc/1
## 複雜查詢很多操作---》es的操作，最重要的就是查詢
# es使用場景---》搜尋功能用----》原來資料在mysql中----》同步到es中---》以後只要mysql新增記錄，都同步到es中---》搜尋的時候，去es中搜索---》返回給前端，完成搜尋功能
2 Elasticsearch之查詢的兩種方式
# 方式一：查詢字串(用的少)
GET lqz/_doc/_search?q=from:gu
GET lqz/_doc/_search?q=age:30
# 方式二：結構化查詢
GET lqz/_doc/_search
{
"query": {
"match": {
"from": "gu"
}
}
}
GET lqz/_search
{
"query": {
"match": {
"age": 30
}
}
}
3 term與match查詢
# 結構化查詢
# match 和 term的區別 面試會問？
# match：會對搜尋的關鍵詞進行分詞，按分詞去搜索
# term:不會對搜尋的關鍵字進行分詞，而直接搜尋，精準匹配
### term不分詞，精準搜尋----》武器很長---》搜不到結果
-存資料的時候，用了分詞 【確實 很 黃，武器 很 長，性格 很 直】
### match分詞 武器 長 ---》有結果
### 用的最多的都是分詞的搜法---》分詞的粒度 ----》分詞器決定的（存的分詞，搜的分詞）
{
"query": {
"match": {
"age": 30
}
}
}
# match_all 查詢所有
GET lqz/_search
{
"query": {
"match_all": {
}
}
}
## match_phrase 短語查詢
GET t1/_doc/_search
{
"query": {
"match_phrase": {
"title": {
"query": "中國",
"slop": 4
}
}
}
}
# slop 把中國分詞後，每個詞之間有0--4個文字都能搜出來
## match_phrase_prefix 最左字首查詢
GET t3/_doc/_search
{
"query": {
"match_phrase_prefix": {
"desc": "bea"
}
}
}
## 多條件查詢-->不能在match中加多個條件
GET lqz/_doc/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"from": "gu"
}
},
{
"match": {
"age": "30"
}
}
]
}
}
}
4 Elasticsearch之排序查詢
GET lqz/_doc/_search
{
"query": {
"match": {
"from": "gu"
}
},
"sort": [
{
"age": {
"order": "desc"
}
}
]
}
## desc 降序 asc 升序
## 不是什麼資料型別都能排序
# 只支援數字和時間型別
5 Elasticsearch之分頁查詢
# "from": 2, 從第幾條開始
# "size": 1 取幾條
GET lqz/_doc/_search
{
"query": {
"match_all": {}
},
"from": 2,
"size": 1
}
6 Elasticsearch之布林查詢
# must（and） 與的條件
GET lqz/_doc/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"from": "gu"
}
}
]
}
}
}
# should（or） 或者條件
GET lqz/_doc/_search
{
"query": {
"bool": {
"should": [
{
"match": {
"from": "gu"
}
},
{
"match": {
"age": 18
}
}
]
}
}
}
# must_not（not） 取反
GET lqz/doc/_search
{
"query": {
"bool": {
"must_not": [
{
"match": {
"from": "gu"
}
},
{
"match": {
"age": 18
}
}
]
}
}
}
# filter
GET lqz/doc/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"from": "gu"
}
}
],
"filter": {
"range": {
"age": {
"gt": 25
}
}
}
}
}
}
# gt gte lt lte 用法
7 Elasticsearch之查詢結果過濾
GET lqz/_doc/_search
{
"query": {
"match_all": {
}
},
"_source": ["name", "age"]
}
8 Elasticsearch之高亮查詢
### 預設高亮樣式
GET lqz/_doc/_search
{
"query": {
"match": {
"name": "石頭"
}
},
"highlight": {
"fields": {
"name": {}
}
}
}
### 自定義高亮樣式
GET lqz/_doc/_search
{
"query": {
"match": {
"desc": "貌美"
}
},
"highlight": {
"pre_tags": "<b class='key' style='color:red'>",
"post_tags": "</b>",
"fields": {
"desc": {}
}
}
}
9 Elasticsearch之聚合函式
# avg max min sum
select max(age) as my_avg
GET lqz/_doc/_search
{
"query": {
"match_all": {
}
},
"aggs": {
"my_max": {
"max": {
"field": "age"
}
}
},
"_source": ["name", "age"]
}
GET lqz/doc/_search
{
"query": {
"match": {
"from": "gu"
}
},
"aggs": {
"my_max": {
"max": {
"field": "age"
}
}
},
"size": 0
}
GET lqz/_doc/_search
{
"size": 0,
"query": {
"match_all": {}
},
"aggs": {
"age_group": {
"range": {
"field": "age",
"ranges": [
{
"from": 0,
"to": 26
},
{
"from": 26,
"to": 31
}
]
}
}
}
}
10 python 操作es
# elaticsearch:官方提供的，類似於原生操作，pymysql
https://github.com/elastic/elasticsearch-py
from elasticsearch import Elasticsearch
# 純用requests模組，也可以實現
# import requests
# res=requests.put("http://localhost:9200/lqz5")
# print(res)
# 服務端 7.0.5版本 7的版本都行
# pip3 install elasticsearch==7.0.5
client=Elasticsearch("http://localhost:9200")
# 建立索引（Index）
# result = client.indices.create(index='user',ignore=400)
# print(result)
# 刪除索引
# result = client.indices.delete(index='lqz3')
# print(result)
# 插入資料
'''
PUT news/_doc/1
{
"userid":"1",
"username":lqz,
"password":"123",
}
'''
# 把mysql的資料，同步到es中--》pymsql開啟查詢---》直接存到es中
# data = {'userid': '1', 'username': 'lqz','password':'123'}
# result = client.create(index='news', doc_type='_doc', id=1, body=data)
# print(result)
# 更新資料
'''
不用doc包裹會報錯
ActionRequestValidationException[Validation Failed: 1: script or doc is missing
'''
# data ={'doc':{'userid': '1', 'username': 'lqz','password':'123ee','test':'test'}}
# result = client.update(index='news', doc_type='_doc', body=data, id=1)
# print(result)
# 刪除資料
# result = client.delete(index='news', doc_type='_doc', id=1)
# print(result)
# 查詢
# 查詢所有文件
# query = {'query': {'match_all': {}}}
# 查詢名字叫做jack的所有文件
# query = {'query': {'term': {'title': '國'}}}
query = {'query': {'match': {'title': '中國'}}}
# 查詢年齡大於11的所有文件
# query = {'query': {'range': {'age': {'gt': 11}}}}
allDoc = client.search(index='t1',body=query)
print(allDoc)
# elaticsearch-dsl：高階庫，類似於orm，django的orm一樣
from datetime import datetime
from elasticsearch_dsl import Document, Date, Nested, Boolean, analyzer, InnerDoc, Completion, Keyword, Text, Integer
from elasticsearch_dsl.connections import connections
connections.create_connection(hosts=["localhost"])
class Article(Document):
title = Text(analyzer='ik_max_word', search_analyzer="ik_max_word", fields={'title': Keyword()})
author = Keyword()
class Index:
name = 'myindex'
# def save(self, **kwargs):
# return super(Article, self).save(**kwargs)
if __name__ == '__main__':
# Article.init() # 建立對映
# 儲存資料
# article = Article()
# article.title = "python測試開發崗位"
# article.author = "lqz"
# article.save() # 資料就儲存了
# 查詢資料
# s=Article.search()
# s = s.filter('match', title="開發")
# results = s.execute()
# print(results[0].title)
# 刪除資料
# s = Article.search()
# s = s.filter('match', title="開發").delete()
# print(s)
# 修改資料
s = Article().search()
s = s.filter('match', title="測試")
results = s.execute()
print(results[0])
results[0].title="測試開發"
results[0].save()
ik分詞器：中文分詞器---》壓縮包---》解壓後放到es的---》重啟即可---》支援ik分詞--》符合果然習慣
ik有兩種分詞方式：
-ik_smart:分的詞少
-ik_max_word：分的詞會多
11 搭建叢集es
es使用兩種不同的方式來發現對方：
廣播--->在同一個網路中，只要開啟多個es例項（用的少）
單播--->指定跟誰誰誰組成叢集
# 4 臺機器的叢集
###機器一配置
cluster.name: my_es1
node.name: node1
network.host: 127.0.0.1
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: ["127.0.0.1:9300", "127.0.0.1:9302", "127.0.0.1:9303", "127.0.0.1:9304"]
cluster.initial_master_nodes: ["127.0.0.1:9300"]
http.cors.enabled: true
http.cors.allow-origin: "*"
### 機器2配置
cluster.name: my_es1
node.name: node2
network.host: 127.0.0.1
http.port: 9202
transport.tcp.port: 9302
node.master: true
node.data: true
discovery.zen.ping.unicast.hosts: ["127.0.0.1:9300", "127.0.0.1:9302", "127.0.0.1:9303", "127.0.0.1:9304"]
http.cors.enabled: true
http.cors.allow-origin: "*"
###機器3配置
cluster.name: my_es1
node.name: node3
network.host: 127.0.0.1
http.port: 9203
transport.tcp.port: 9303
discovery.zen.ping.unicast.hosts: ["127.0.0.1:9300", "127.0.0.1:9302", "127.0.0.1:9303", "127.0.0.1:9304"]
http.cors.enabled: true
http.cors.allow-origin: "*"
### 機器4配置
cluster.name: my_es1
node.name: node4
network.host: 127.0.0.1
http.port: 9204
transport.tcp.port: 9304
node.master: true
node.data: true
discovery.zen.ping.unicast.hosts: ["127.0.0.1:9300", "127.0.0.1:9302", "127.0.0.1:9303", "127.0.0.1:9304"]
http.cors.enabled: true
http.cors.allow-origin: "*"
# 腦裂問題
discovery.zen.minimum_master_nodes: 3 # 叢集節點個數除以2+1