1. 环境说明

ELK已部署如下：

[root@DoM01 ~]# kubectl get service -n elk
NAME                            TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)             AGE
elasticsearch                   NodePort    10.1.4.111     <none>        9200:30028/TCP      2d
elasticsearch-master            ClusterIP   10.1.104.188   <none>        9200/TCP,9300/TCP   30h
elasticsearch-master-headless   ClusterIP   None           <none>        9200/TCP,9300/TCP   30h
kibana-kibana                   NodePort    10.1.43.153    <none>        5601:30026/TCP      2d
logstash-logstash-headless      ClusterIP   None           <none>        9600/TCP            2d

2. 收集方案

2.1 方案说明

1）创建一个pod，其中包含两个容器，分别是主服务（以nginx为例）和filebeat
2）filebeat和nginx共享日志目录，nginx写入filebeat收集
3）将filebeat的配置文件挂载为configmap便于修改

2.2 方案示例

yml文件

apiVersion: v1
kind: Pod
metadata:
  name: nginx-filebeat
  namespace: test
spec:
  containers:
  - name: nginx
    image: harbocto.xxx.com.cn/public/nginx
    env:
      - name: TZ
        value: Asia/Shanghai
    volumeMounts:
    - mountPath: /var/log/nginx/
      name: nginxlog
  - name: filebeat
    image: harbocto.xxx.com.cn/public/filebeat:8.0.0-SNAPSHOT
    env:
      - name: TZ
        value: Asia/Shanghai
    volumeMounts:
    - mountPath: /var/log/nginx/
      name: nginxlog
    - mountPath: /usr/share/filebeat/filebeat.yml
      name: filebeatyml
      subPath: filebeat.yml
  volumes:
  - name: nginxlog
    persistentVolumeClaim:
      claimName:  nginxlog
  - name: filebeatyml
    configMap:
      name: filebeat
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat
  namespace: test
data:
  filebeat.yml: |
    filebeat.inputs:
    - type: log
      enabled: true
      paths:
        - /var/log/nginx/*.log
    setup.ilm.enabled: false
    setup.template.name: "nginx-dev"
    setup.template.pattern: "nginx-dev-*"
    output.elasticsearch:
      host: 'nginx_test'
      hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master.elk:9200}'
      index: "nginx-dev-%{+yyyy.MM.dd}"
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: nginxlog
  namespace: test
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi

启动

[root@DoM01 ~]# kubectl create namespace test
[root@DoM01 ~]# kubectl create -f nginx-filebeat-test.yml
pod/nginx-filebeat created
configmap/filebeat created
[root@DoM01 ~]# kubectl get pod -n test
NAME             READY   STATUS    RESTARTS   AGE
nginx-filebeat   2/2     Running   0          35m

2.3 filebeat的配置文件

上边comfigmap的说明

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat
  namespace: test
data:
  # filebeat.yml是filebeat的配置文件名（关于key名参见《k8s对象-congmap》，此处不赘述）
  filebeat.yml: |
    filebeat.inputs:
    - type: log
      enabled: true
      # 指明收集日志的目录
      paths:
        - /var/log/nginx/*.log
    #索引生命管理必须关闭，否则忽略自定义索引
    setup.ilm.enabled: false
    #自定义索引模板
    setup.template.name: "nginx-dev"
    setup.template.pattern: "nginx-dev-*"
    output.elasticsearch:
      # 链接elasticsearch，前文环境说明里可以看到改url
      hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master.elk:9200}'
      #自定义索引
      index: "nginx-dev-%{+yyyy.MM.dd}"
      index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}