def __init__(self, rules, args=None):
super(BlacklistV2Rule, self).__init__(rules, args=None)
self.expand_entries('blacklist_v2')
def compare(self, event):
term = lookup_es_key(event, self.rules['compare_key'])
# 循环配置文件， 这种做法对性能有一定的损耗，在没找到更合适的解决方案前，就采取这种方式
for i in self.rules['blacklist_v2']:
if i in term:
return True
return False
elastalert/config.py
# 新增
rules_mapping = {
'frequency': ruletypes.FrequencyRule,
'any': ruletypes.AnyRule,
'spike': ruletypes.SpikeRule,
'blacklist': ruletypes.BlacklistRule,
'blacklist_v2': ruletypes.BlacklistV2Rule,
'whitelist': ruletypes.WhitelistRule,
'change': ruletypes.ChangeRule,
'flatline': ruletypes.FlatlineRule,
'new_term': ruletypes.NewTermsRule,
'cardinality': ruletypes.CardinalityRule,
'metric_aggregation': ruletypes.MetricAggregationRule,
'percentage_match': ruletypes.PercentageMatchRule,
}
elastalert/schema.yaml
# 新增
- title: BlacklistV2
required: [blacklist_v2, compare_key]
properties:
type: {enum: [blacklist_v2]}
compare_key: {'items': {'type': 'string'},'type': ['string', 'array']}
blacklist: {type: array, items: {type: string}}
打进Docker
做了个简单DockerFile做参考
FROM python:2.7-alpine
ENV SITE_PACKAGES /usr/local/lib/python2.7/site-packages/elastalert
WORKDIR /opt/elastalert
RUN apk update && apk add gcc ca-certificates openssl-dev openssl libffi-dev gcc musl-dev tzdata openntpd && \
pip install elastalert && cp -rf /usr/share/zoneinfo/Asia/Taipei /etc/localtime
COPY ./ /opt/elastalert
CMD ["/opt/elastalert/start.sh"]
start.sh
#!/bin/sh
SITE_PATH=/usr/local/lib/python2.7/site-packages/elastalert
CONFIG=/opt/elastalert/config/config.yaml
MODULES=/opt/elastalert/modules
if [ -n "${MODULES}" ]
then
\cp -rf ${MODULES} ${SITE_PATH}
echo "[-] Copy ${MODULES} to ${SITE_PATH}"
fi
\cp -rf elastalert/* ${SITE_PATH}/
echo "[-] Copy elastalert/* to ${SITE_PATH}"
python -m elastalert.elastalert --verbose --config ${CONFIG}
基础工作准备就绪，加入Bee容器管理平台完成自动构建。
实现效果
碰到的坑
Zookeeper
问题描述
老版Kafaka依赖Zookeeper，默认安装时注册地址为:localhost，导致问题的现象：
filebeat错误日志
2018-04-25T09:14:55.590+0800 INFO kafka/log.go:36 client/metadata fetching metadata for [[[system_log] kafkaNode:9092]] from broker %!s(MISSING)
2018-04-25T09:14:55.591+0800 INFO kafka/log.go:36 producer/broker/[[0]] starting up
2018-04-25T09:14:55.591+0800 INFO kafka/log.go:36 producer/broker/[[0 %!d(string=system_log) 0]] state change to [open] on %!s(MISSING)/%!d(MISSING)
2018-04-25T09:14:55.591+0800 INFO kafka/log.go:36 producer/leader/[[system_log %!s(int32=0) %!s(int32=0)]]/%!d(MISSING) selected broker %!d(MISSING)
2018-04-25T09:14:55.591+0800 INFO kafka/log.go:36 producer/leader/[[system_secure %!s(int32=0) %!s(int=3)]]/%!d(MISSING) state change to [retrying-%!d(MISSING)]
2018-04-25T09:14:55.591+0800 INFO kafka/log.go:36 producer/leader/[[system_secure %!s(int32=0) %!s(int32=0)]]/%!d(MISSING) abandoning broker %!d(MISSING)
2018-04-25T09:14:55.592+0800 INFO kafka/log.go:36 producer/broker/[[0]] shut down