##rsyslog
#### RULES ####
$template DynaFile,"/data/log/net/other/%FROMHOST-IP%_%$YEAR%-%$MONTH%-%$DAY%"
$template myformat,"%$NOW% %TIMESTAMP:8:15% %hostname% %syslogtag% %msg%\n"
$ActionFileDefaultTemplate myformat
$template hq,"/data/log/net/HQ/%FROMHOST-IP%_%$YEAR%-%$MONTH%-%$DAY%"
:fromhost-ip, isequal, "127.0.0.1" ?Local
& ~
:hostname,contains,"CC" ?cc
& ~
:hostname,startswith,"HQ" ?hq
& ~
:syslogtag,startswith,"HQ" ?hq
& ~
:FROMHOST-IP,startswith,"10.32.4." ?hq
& ~
:FROMHOST-IP,startswith,"10.32.5" ?hq
& ~
*.* -?DynaFile
& ~
##filebeat
cat /etc/yum.repos.d/filebeat.repo
[filebeat]
name=Elasticsearch repository for 88888888.x packages
baseurl=https://artifacts.elastic.co/packages/8.x/yum
gpgcheck=0
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
#
yum install filebeat -y
#
##3、filebeat 配置
filebeat.inputs:
- type: log
enabled: true
paths:
- /data/log/other/10.1.1.1*
#- /data/log/net/FW/10.1.1.1*
fields:
device_model: "zb-caiwuwww-92"
# kafka_topic: "zb-zhuanxianfw-1-1-1-1-topic"
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.template.enabled: true
setup.template.fields: fields.yml
setup.template.overwrite: true
processors:
- drop_fields:
fields: ['agent', 'ecs', 'beat', 'input_type', 'tags', 'count', '@version', 'log', 'offset', 'type', 'host']
ignore_missing: false
#输出到logstash
output.logstash:
hosts: ["10.1.1.1:5044"]
output.kafka:
enabled: true
hosts: ["10.1.1.1:9092","10.1.1.2:9092","10.1.1.3:9092"]
#hosts: ["10.1.1.1:9092"]
topic: "%{[fields.kafka_topic]}"
compression: gzip
max_message_bytes: 1000000
