Untitled

# cat /etc/app/fluent-bit.conf

[SERVICE]                                                                                             
    Flush        1
    HTTP_Server  On
    Log_Level    info
    Health_Check On
    Parsers_File /etc/app/parsers.conf

[INPUT]
    Name              tail
    Tag               kube.*
    Path              /var/log/containers/*.log
    DB                /var/tmp/fluent-bit.pos
    DB.Sync           Off
    DB.locking        true
    Buffer_Chunk_Size 512k
    Buffer_Max_Size   16384k
    Mem_Buf_Limit     8000m
    Refresh_Interval  25
    Parser            cri

[FILTER]
    Name                kubernetes
    Match               kube.*
    Kube_URL            https://kubernetes.default.svc:443
    Buffer_Size         0
    Kube_Tag_Prefix     kube.var.log.containers.
    Merge_Log           On
    Merge_Parser        catchall
    Keep_Log            Off
    K8S-Logging.Parser  On
    K8S-Logging.Exclude On
    Annotations         Off

[FILTER]
    Name         nest
    Match        kube.*
    Operation    lift
    Nested_under kubernetes
    Add_prefix   kubernetes.

[FILTER]
    Name         nest
    Match        kube.*
    Operation    lift
    Nested_under kubernetes.labels
    Add_prefix   kubernetes.labels.

[FILTER]
    Name         modify
    Match        kube.*
    Rename       kubernetes.labels.app.kubernetes.io/name kubernetes.labels.name
    Rename       kubernetes.labels.app.kubernetes.io/instance kubernetes.labels.instance
#    Rename       kubernetes.labels.helm.sh/chart kubernetes.labels.chart # probably unused, return later
    Remove       kubernetes.docker_id
    Remove       kubernetes.container_hash
    Remove       kubernetes.labels.myteam
    Remove       kubernetes.labels.pod-template-hash
    Remove       kubernetes.labels.canaryversion
    Remove       kubernetes.pod_id
    Remove       kubernetes.labels.controller-revision-hash
    Remove_regex .*kubernetes.labels.helm.sh.*
    Remove_regex .*kubernetes.io.*
    Copy         kubernetes.namespace_name project

[FILTER]
    Name         lua
    Match        kube.*
    Type_int_key timestamp
    script       /etc/app/array_create.lua
    call         array_create

#    [OUTPUT]
#        Name   stdout
#        Match  kube.*
#        # Format json

#    [OUTPUT]
#        Name null
#        Match *

[OUTPUT]
    Name                                 kafka
    Match                                kube.*
    Brokers                              mq1:9092,mq2:9092,mq3:9092
    Topics                               common
    Topic_Key                            kubernetes.labels.bunit
    Dynamic_Topic                        on
    rdkafka.log.connection.close         false
    rdkafka.request.required.acks        0
    rdkafka.queue.buffering.max.messages 500000
    rdkafka.compression.codec            snappy
    rdkafka.message.max.bytes            4000000
    rdkafka.batch.num.messages           20000
    rdkafka.batch.size                   2000000
    rdkafka.queue.buffering.max.ms       8
    workers                              8