From 452203ac993887dc365ac6551a838788c66643fa Mon Sep 17 00:00:00 2001
From: Steve Wilkerson <wilkers.steve@gmail.com>
Date: Mon, 14 Jan 2019 16:16:21 -0600
Subject: [PATCH] Fluent-Logging: Update configuration

This updates the configuration for the fluent-logging chart to
include inputs for libvirt and qemu, as well as updates the output
configurations to include the appropriate buffer configuration
overrides

Change-Id: Id908ea875b07004202795dac68df7e9727ae9771
---
 .../osh-infra-logging/fluent-logging.yaml     | 102 +++++++++++++-----
 1 file changed, 75 insertions(+), 27 deletions(-)

diff --git a/global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml b/global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml
index f047d6e3c..1315171d8 100644
--- a/global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml
+++ b/global/software/charts/osh-infra/osh-infra-logging/fluent-logging.yaml
@@ -177,6 +177,36 @@ data:
             DB.Sync: Normal
             Buffer_Chunk_Size: 1M
             Buffer_Max_Size: 1M
+        - libvirt:
+            header: input
+            Name: tail
+            Tag: libvirt
+            Path: /var/log/libvirt/libvirtd.log
+            DB: /var/log/libvirt.db
+            Mem_Buf_Limit: 5MB
+            DB.Sync: Normal
+            Buffer_Chunk_Size: 1M
+            Buffer_Max_Size: 1M
+        - qemu:
+            header: input
+            Name: tail
+            Tag: qemu
+            Path: /var/log/libvirt/qemu/*.log
+            DB: /var/log/qemu.db
+            Mem_Buf_Limit: 5MB
+            DB.Sync: Normal
+            Buffer_Chunk_Size: 1M
+            Buffer_Max_Size: 1M
+        - libvirt_record_modifier:
+            header: filter
+            Name: record_modifier
+            Match: libvirt
+            Record: hostname ${HOSTNAME}
+        - qemu_record_modifier:
+            header: filter
+            Name: record_modifier
+            Match: qemu
+            Record: hostname ${HOSTNAME}
         - kernel_record_modifier:
             header: filter
             Name: record_modifier
@@ -297,11 +327,15 @@ data:
             port: "#{ENV['ELASTICSEARCH_PORT']}"
             logstash_format: true
             logstash_prefix: journal
-            buffer_chunk_limit: 2M
-            buffer_queue_limit: 512
-            flush_interval: "10"
-            max_retry_wait: 300
-            disable_retry_limit: ""
+            buffer:
+              -
+                - header: buffer
+                  flush_thread_count: 8
+                  flush_interval: "15"
+                  chunk_limit_size: 8MB
+                  queue_limit_length: 256
+                  retry_max_interval: 30
+                  retry_forever: false
         - kernel_elasticsearch:
             header: match
             type: elasticsearch
@@ -313,11 +347,15 @@ data:
             port: "#{ENV['ELASTICSEARCH_PORT']}"
             logstash_format: true
             logstash_prefix: kernel
-            buffer_chunk_limit: 2M
-            buffer_queue_limit: 512
-            flush_interval: "10"
-            max_retry_wait: 300
-            disable_retry_limit: ""
+            buffer:
+              -
+                - header: buffer
+                  flush_thread_count: 8
+                  flush_interval: "15"
+                  chunk_limit_size: 8MB
+                  queue_limit_length: 256
+                  retry_max_interval: 30
+                  retry_forever: false
         - ceph_elasticsearch:
             header: match
             type: elasticsearch
@@ -329,12 +367,15 @@ data:
             port: "#{ENV['ELASTICSEARCH_PORT']}"
             logstash_format: true
             logstash_prefix: ceph
-            buffer_chunk_limit: 10M
-            buffer_queue_limit: 512
-            flush_interval: "10"
-            max_retry_wait: 300
-            disable_retry_limit: ""
-            num_threads: 8
+            buffer:
+              -
+                - header: buffer
+                  flush_thread_count: 8
+                  flush_interval: "15"
+                  chunk_limit_size: 8MB
+                  queue_limit_length: 256
+                  retry_max_interval: 30
+                  retry_forever: false
         - oslo_fluentd_elasticsearch:
             header: match
             type: elasticsearch
@@ -346,13 +387,15 @@ data:
             port: "#{ENV['ELASTICSEARCH_PORT']}"
             logstash_format: true
             logstash_prefix: openstack
-            buffer_type: memory
-            buffer_chunk_limit: 10M
-            buffer_queue_limit: 512
-            flush_interval: "10"
-            max_retry_wait: 300
-            request_timeout: 60
-            disable_retry_limit: ""
+            buffer:
+              -
+                - header: buffer
+                  flush_thread_count: 8
+                  flush_interval: "15"
+                  chunk_limit_size: 8MB
+                  queue_limit_length: 256
+                  retry_max_interval: 30
+                  retry_forever: false
             type_name: oslo_openstack_fluentd
         - elasticsearch:
             header: match
@@ -364,11 +407,16 @@ data:
             host: "#{ENV['ELASTICSEARCH_HOST']}"
             port: "#{ENV['ELASTICSEARCH_PORT']}"
             logstash_format: true
-            buffer_chunk_limit: 10M
-            buffer_queue_limit: 512
+            buffer:
+              -
+                - header: buffer
+                  flush_thread_count: 8
+                  flush_interval: "15"
+                  chunk_limit_size: 8MB
+                  queue_limit_length: 256
+                  retry_max_interval: 30
+                  retry_forever: false
             flush_interval: "10"
-            max_retry_wait: 300
-            disable_retry_limit: ""
       parsers:
         - syslog:
             header: parser