Skip to content

Commit f1aac12

Browse files
kvvitkvvit
committed
Enable logstash
1 parent f066edc commit f1aac12

File tree

5 files changed

+170
-12
lines changed

5 files changed

+170
-12
lines changed

‎terraform/modules/k8s-addons/eks-logstash.tf

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
locals {
2+
logstash = merge(
3+
local.helm_releases[index(local.helm_releases.*.id, "logstash")],
4+
var.logstash)
5+
}
6+
7+
data "template_file" "logstash" {
8+
template = file("${path.module}/elk-templates/logstash-values.yaml")
9+
vars = {
10+
env = "${local.env}"
11+
}
12+
}
13+
14+
resource "helm_release" "logstash" {
15+
count = local.logstash.enabled ? 1 : 0
16+
17+
name = "logstash"
18+
chart = "logstash"
19+
repository = local.logstash.repository
20+
version = local.logstash.chart_version
21+
22+
namespace = "elk"
23+
timeout = "900"
24+
max_history = var.helm_release_history_size
25+
26+
values = compact(
27+
[data.template_file.logstash.rendered,
28+
try(local.logstash.helm_values_override, null),
29+
])
30+
}
31+

‎terraform/modules/k8s-addons/elk-templates/filebeat-values.yaml

+5-12
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,17 @@ filebeatConfig:
1818
matchers:
1919
- logs_path:
2020
logs_path: "/var/log/containers/"
21-
- drop_event:
22-
when:
23-
equals:
24-
kubernetes.container.name: "filebeat"
21+
- drop_event.when.or:
22+
- equals.kubernetes.container.name: "filebeat"
23+
- equals.kubernetes.container.name: "logstash"
2524
- drop_event.when.and:
2625
- equals.k8s-app: "coredns"
2726
- regexp.message: ".*NOERROR.*"
2827
setup.template.name: "filebeat"
2928
setup.template.pattern: "filebeat-*"
3029
setup.ilm.enabled: false
31-
output.elasticsearch:
32-
username: '$${ELASTICSEARCH_USERNAME}'
33-
password: '$${ELASTICSEARCH_PASSWORD}'
34-
protocol: https
35-
hosts: ["$${ELASTICSEARCH_HOST}:9200"]
36-
ssl.verification_mode: none
37-
indices:
38-
- index: "filebeat-%%{+yyyy.MM.dd}-000001"
30+
output.logstash:
31+
hosts: ["logstash-logstash.elk.svc:5044"]
3932
extraEnvs:
4033
- name: 'ELASTICSEARCH_USERNAME'
4134
valueFrom:

‎terraform/modules/k8s-addons/elk-templates/logstash-values.yaml

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
replicas: 1
2+
3+
resources:
4+
requests:
5+
cpu: 100m
6+
memory: 1Gi
7+
limits:
8+
memory: 2Gi
9+
10+
11+
# persistence:
12+
# enabled: true
13+
14+
logstashConfig:
15+
logstash.yml: |
16+
http.host: 0.0.0.0
17+
xpack.monitoring.enabled: true
18+
xpack.monitoring.elasticsearch.username: '$${ELASTICSEARCH_USERNAME}'
19+
xpack.monitoring.elasticsearch.password: '$${ELASTICSEARCH_PASSWORD}'
20+
xpack.monitoring.elasticsearch.ssl.certificate_authority: '/etc/logstash/certificates/ca.crt'
21+
xpack.monitoring.elasticsearch.hosts: ["https://elasticsearch-es-internal-http.elk.svc:9200"]
22+
23+
logstashPipeline:
24+
logstash.conf: |
25+
input {
26+
tcp {
27+
port => 5400
28+
codec => json
29+
}
30+
beats {
31+
port => 5044
32+
}
33+
}
34+
35+
output {
36+
elasticsearch {
37+
hosts => ["elasticsearch-es-internal-http.elk.svc:9200"]
38+
user => '$${ELASTICSEARCH_USERNAME}'
39+
password => '$${ELASTICSEARCH_PASSWORD}'
40+
index => "filebeat-%%{+yyyy.MM.dd}-000001"
41+
manage_template => true
42+
template => '/etc/elk/logstash-index-template.json'
43+
template_name => 'filebeat'
44+
template_overwrite => true
45+
action => "create"
46+
ssl => true
47+
ssl_certificate_verification => false
48+
cacert => '/etc/logstash/certificates/ca.crt'
49+
}
50+
stdout { codec => rubydebug }
51+
}
52+
53+
# https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html
54+
55+
secrets:
56+
- name: "index-template"
57+
value:
58+
logstash-index-template.json: |
59+
{
60+
"index_patterns": ["filebeat-*"],
61+
"data_stream": {},
62+
"template": {
63+
"settings": {
64+
"index.refresh_interval": "5s",
65+
"index.mapping.total_fields.limit": 5000,
66+
"index.lifecycle.name": "filebeat-policy",
67+
"number_of_shards": 1,
68+
"number_of_replicas": 1
69+
}
70+
}
71+
}
72+
73+
secretMounts:
74+
- name: logstash-index-template
75+
secretName: logstash-logstash-index-template
76+
path: /etc/elk
77+
78+
extraPorts:
79+
- name: beats
80+
containerPort: 5044
81+
- name: tcp
82+
containerPort: 5400
83+
84+
service:
85+
ports:
86+
- name: beats
87+
port: 5044
88+
protocol: TCP
89+
targetPort: 5044
90+
- name: tcp
91+
port: 5400
92+
protocol: TCP
93+
targetPort: 5400
94+
95+
extraEnvs:
96+
- name: 'ELASTICSEARCH_USERNAME'
97+
valueFrom:
98+
secretKeyRef:
99+
name: elastic-credentials
100+
key: username
101+
- name: 'ELASTICSEARCH_PASSWORD'
102+
valueFrom:
103+
secretKeyRef:
104+
name: elastic-credentials
105+
key: password
106+
- name: 'ELASTICSEARCH_HOST'
107+
valueFrom:
108+
secretKeyRef:
109+
name: elastic-credentials
110+
key: host
111+
112+
extraVolumes:
113+
- name: ca-certs
114+
secret:
115+
secretName: elasticsearch-es-http-certs-public
116+
117+
extraVolumeMounts:
118+
- name: ca-certs
119+
mountPath: /etc/logstash/certificates
120+
readOnly: true
121+

‎terraform/modules/k8s-addons/helm-releases.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -101,3 +101,9 @@ releases:
101101
repository: https://helm.elastic.co/
102102
chart_version: 7.17.3
103103
namespace: elk
104+
- id: logstash
105+
enabled: true
106+
chart: elastic/logstash
107+
repository: https://helm.elastic.co/
108+
chart_version: 7.17.3
109+
namespace: elk

‎terraform/modules/k8s-addons/variables.tf

+7
Original file line numberDiff line numberDiff line change
@@ -119,3 +119,10 @@ variable "elk" {
119119
default = {}
120120
description = "ELK configuration"
121121
}
122+
123+
variable "logstash" {
124+
type = any
125+
default = {}
126+
description = "Logstash configuration"
127+
}
128+

0 commit comments

Comments
 (0)