Saltstack Automatic grouping
阿新 • • 發佈：2017-08-28
saltstack automatic grouping
一、要點：
知道key驗證存放的目錄
key在驗證是手動還自動
對不在線的主機的處理
minion_id的命名規範
二、使用的技術棧
saltstack 相關的庫：salt.config,salt.client,salt.runner
使用redis 存放兩個數據庫，第一個為存為字典，用於存放minion_id與物理IP的對應，另一個存為集合，用於項目_業務命名的方式包含相應的主機
使用watchdog對目錄變更監控，事件觸發機制
三、完整代碼
# coding:utf8
# Created by IntelliJ IDEA.
# User: King.gp
# Date: 2016/10/9
# Time: 10:55
# To change this template use File | Settings | File Templates.
import time
import re
import sys
import redis
import logging
import salt.config as saltc
import salt.runner as saltr
import salt.client as SaltC
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
logging.basicConfig(level=logging.DEBUG,
format=‘%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s‘,
datefmt=‘%a, %d %b %Y %H:%M:%S‘,
filename=‘myapp.log‘, )
#################################################################################################
# 定義一個StreamHandler，將INFO級別或更高的日誌信息打印到標準錯誤，並將其添加到當前的日誌處理對象#
# console = logging.StreamHandler()
# console.setLevel(logging.INFO)
# formatter = logging.Formatter(‘%(name)-12s: %(levelname)-8s %(message)s‘)
# console.setFormatter(formatter)
# logging.getLogger(‘‘).addHandler(console)
################################################################################################
class MonitorHandler(FileSystemEventHandler):
# time
localtime = time.asctime(time.localtime(time.time()))
# 文件或目錄被移動
def on_moved(self, event):
logging.info(MonitorHandler.localtime + " - Event :文件/目錄： {0} 被移動!".format(event.src_path))
call_func()
# 文件或目錄被創建
def on_created(self, event):
logging.info(MonitorHandler.localtime + " - Event: 文件/目錄: {0} 被創建".format(event.src_path))
call_func()
# 文件或目錄被刪除
def on_deleted(self, event):
logging.info(MonitorHandler.localtime + " - Event :文件/目錄： {0} 被刪除!".format(event.src_path))
call_func()
# 文件或目錄被修改
def on_modified(self, event):
logging.info(MonitorHandler.localtime + " - Event :文件/目錄： {0} 被修改!".format(event.src_path))
call_func()
def client():
slc = SaltC.LocalClient()
return slc
class NodeGroup(object):
def __init__(self):
self.member = redis.StrictRedis(host=‘127.0.0.1‘,
port=6379,
db=6, ) # 字典數據
self.project = redis.StrictRedis(host=‘127.0.0.1‘,
port=6379,
db=7, ) # 集合數據
def update_info(self):
self.member.flushdb()
self.project.flushdb()
with open(‘host‘, ‘rb‘) as fd:
lines = fd.readlines()
pattern = re.compile(r‘[a-z]+‘)
for line in lines:
line = line.strip().split() # 1.2.3.4 web01.crms.com
self.member.set(line[1], line[0]) # web01.crms.com
profession_number = line[1].split(".")[0] # web01
m = re.match(pattern, profession_number)
if m:
profession = m.group()
project_suffix = line[1].split(‘.‘)[-2:]
project = ‘.‘.join(project_suffix) # crms.com
self.project.sadd(‘_‘.join([project, profession]), ‘‘) # crms.com_web: ‘‘
member_list = []
match_dict = {}
for category in self.project.keys():
category_pattern = re.compile(category.split("_")[1]) # vm, logstash, elasticsearch
pattern_member_list = []
for key in self.member.keys():
# [‘test.logstash.yinker.com‘, ‘test.elastic.yinker.com‘, ‘test.vm.yinker.com‘]
category_match = category_pattern.search(key) # web we01.crms.com
if category_match:
pattern_member_list.append(key)
match_dict[category_pattern.pattern] = pattern_member_list
if match_dict not in member_list:
member_list.append(match_dict)
# members_dict = {}
for p in self.project.keys():
pattern = re.compile(p.split("_")[0])
for k, values in member_list[0].items():
members_list = []
for v in values:
match = pattern.search(v) # suffix ,
if match:
members_list.append(match.string)
else:
continue
# members_dict[‘_‘.join([pattern.pattern, k])] = members_list
# print members_list
self.project.sadd(‘_‘.join([pattern.pattern, k]), members_list)
self.project.srem(‘_‘.join([pattern.pattern, k]), ‘‘)
def append_config(self):
with open("/etc/salt/master.d/nodegroups.conf", "w") as fd:
fd.write("nodegroups:\n")
with open("/etc/salt/master.d/nodegroups.conf", "a+")as f:
for mem in self.project.keys():
# zookeeperg0: ‘[email protected]
# ["[‘idp01.crms.com‘,‘idp02.crms.com‘]"]
f.write(" " + mem + ": " + "‘" + "[email protected]" + (‘‘.join(
[i.replace("[‘", ‘‘).replace("‘]", ‘‘).replace("‘", ‘‘) for i in
self.project.smembers(mem)]) + "‘" + ‘\n‘).replace(‘ ‘, ‘‘))
def control_hosts():
priv_opts = saltc.master_config(‘/etc/salt/master‘)
priv_runner = saltr.RunnerClient(priv_opts)
# r = priv_runner()
host_down = priv_runner.cmd("manage.down", ["removekeys=True"]) # 移除DOWN狀態的minion
if len(host_down) > 0:
logging.info(MonitorHandler.localtime + "minion_id: {0} 己經下線 ".format(‘ ‘.join(host_down)))
local = client()
host_items = local.cmd("*", "grains.item", [‘ip4_interfaces‘, ‘id‘])
time.sleep(3)
if host_items:
raw = ‘‘
for v in host_items.values():
raw = ‘{0}{1}\n‘.format(raw, ‘ ‘.join([v[‘ip4_interfaces‘][‘eth0‘][0], v[‘id‘]]))
with open("host", "w+") as f:
f.write(raw)
return True
else:
logging.error(MonitorHandler.localtime + "拉取grains信息失敗")
return False
def call_func():
status = control_hosts()
if status:
group_agg = NodeGroup()
group_agg.update_info()
group_agg.append_config()
else:
logging.error(MonitorHandler.localtime + "拉取grains信息失敗")
sys.exit(status=127)
if __name__ == "__main__":
Monitor_Dir = ‘/etc/salt/pki/master/minions‘
event_handler = MonitorHandler()
observer = Observer()
NodeGroup_File = ‘/etc/salt/master.d/nodegroups.conf‘
observer.schedule(event_handler, path=Monitor_Dir, recursive=True)
observer.start()
try:
while True:
time.sleep(10)
except KeyboardInterrupt:
observer.stop()
observer.join()四、關於缺陷
關於怎麽判斷minion不在線的界定，是因為網絡抖動，還是真的下線;
如果出來這樣的問題是移除，還是不處理，如何取舍，因為這涉及一個問題，利用分組無法準確的執行任務，因為分組依賴於salt的允許的key對應的主機;
啟動時會提示遞歸溢出，所有移除的主機都會輸出到console上;
請自行添加到supervisor中。
本文出自 “和風細雨” 博客，請務必保留此出處http://essun.blog.51cto.com/721033/1959970
Saltstack Automatic grouping