Всем привет!
Подскажите пожалуйста, у меня в nginx пишутся логи в JSON формате далее Fluentbit берёт эти логи и отправляет в Kafka, смотрю что прилетает в Kafka и вижу что Fluentbit мой JSON (из nginx) обернул ещё раз в свой JSON. Получился JSON в JSONе.
Вопрос - как отключить повторную обёртку в JSON. Фактуру приложил. Благодарю.
vi nginx.conf
log_format main_json escape=json '{'
'"@timestamp":$msec,'
'"appName":"pirate",'
'"localTime":"$time_iso8601",'
'"level":"INFO",'
'"text":"client=$remote_addr method=$request_method request=$request request_length=$request_length time_local=$time_local status=$status bytes_sent=$bytes_sent body_bytes_sent=$body_bytes_sent referer=$http_referer remote_user=$remote_user user_agent=$http_user_agent upstream_addr=$upstream_addr upstream_status=$upstream_status request_time=$request_time upstream_response_time=$upstream_response_time upstream_connect_time=$upstream_connect_time upstream_header_time=$upstream_header_time",'
'"eventId":"33637347-6246-7641-3185-75346984632",'
'"appType":"NGINX",'
'"levelInt":1,'
'"loggerName":"io.tracing.internal.reporters.LoggingReporter",'
'"threadName":"http-nio-8080-exec-3",'
'"callerClass":"io.tracing.internal.reporters.LoggingReporter",'
'"callerMethod":"report",'
'"callerLine":"1",'
'"envType":"fluentd",'
'"namespace":"kx-dc",'
'"podName":"mf-nginxhost",'
'"es_index_name_suffix":"kx-dc.nginx",'
'"tslgClientVersion":"0.0.1",'
'"tec":{"nodeName":"mf-nginxhost.test.datacenter.ru",'
'"podIp":"10.64.122.125"}'
'}';
vi fluent-bit.conf
# Fluentbit Config
[SERVICE]
Flush 1
# Parsers_File parsers.conf
[INPUT]
Name tail
Tag nginxaccess.*
Path /app/nginx/logs/*.log
LineFormat json
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 1
[OUTPUT]
Name Kafka
Match *
Topics tslg-app-logs
Timestamp_Key time
Brokers testhost-kafka.test.datacenter.ru:9092
rdkafka.request.required.acks 1
rdkafka.log.connection.close false
rdkafka.compression.codec snappy
rdkafka.security.protocol ssl
rdkafka.ssl.ca.location /opt/fluentbit/etc/fluent-bit/newcert2/ca-kafka-cert.pem
rdkafka.ssl.certificate.location /opt/fluentbit/etc/fluent-bit/newcert2/client-kafka-cert.pem
rdkafka.ssl.key.location /opt/fluentbit/etc/fluent-bit/newcert2/client-kafka-cert-key.pem
в кафку залетаеют JSONы такого формата
{
"time": 1671104247.099066,
"log": "{\"@timestamp\":1671104247.098,\"appName\":\"pirate\",\"localTime\":\"2022-12-15T14:37:27+03:00\",\"level\":\"INFO\",\"text\":\"client\u003d10.186.26.250 method\u003dGET request\u003dGET /api/v1/applic_list/representative/credentials HTTP/2.0 request_length\u003d58 time_local\u003d15/Dec/2022:14:37:27 +0300 status\u003d200 bytes_sent\u003d961 body_bytes_sent\u003d397 referer\u003dhttps://mf-nginxhost.test.datacenter.ru/cash/cash-app/new remote_user\u003d user_agent\u003dMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.106 Safari/537.36 upstream_addr\u003d10.223.18.104:8443 upstream_status\u003d200 request_time\u003d0.198 upstream_response_time\u003d0.199 upstream_connect_time\u003d0.010 upstream_header_time\u003d0.199\",\"eventId\":\"33637347-6246-7641-3185-75346984632\",\"appType\":\"NGINX\",\"levelInt\":1,\"loggerName\":\"io.tracing.internal.reporters.LoggingReporter\",\"threadName\":\"http-nio-8080-exec-3\",\"callerClass\":\"io.tracing.internal.reporters.LoggingReporter\",\"callerMethod\":\"report\",\"callerLine\":\"1\",\"envType\":\"fluentd\",\"namespace\":\"kx-dc\",\"podName\":\"mf-nginxhost\",\"es_index_name_suffix\":\"kx-dc.nginx\",\"tslgClientVersion\":\"0.0.1\",\"tec\":{\"nodeName\":\"mf-nginxhost.test.datacenter.ru\",\"podIp\":\"10.64.122.125\"}}"
}
