From 0061396163493e70e78d0d9aef33fe43032ea55e Mon Sep 17 00:00:00 2001
From: Chris Yoon <chris.cw.yoon@gmail.com>
Date: Mon, 8 Aug 2022 15:37:39 +0900
Subject: [PATCH] opensearch docker compose files added

---
 docker-compose/opensearch/README.md           | 120 ++++++++++++++++++
 docker-compose/opensearch/docker-compose.yml  |  63 +++++++++
 .../opensearch/logstash/config/logstash.yml   |   7 +
 .../logstash/pipeline/logstash.conf           |  21 +++
 .../opensearch/send-message-sample.json       |   1 +
 5 files changed, 212 insertions(+)
 create mode 100644 docker-compose/opensearch/README.md
 create mode 100644 docker-compose/opensearch/docker-compose.yml
 create mode 100644 docker-compose/opensearch/logstash/config/logstash.yml
 create mode 100644 docker-compose/opensearch/logstash/pipeline/logstash.conf
 create mode 100644 docker-compose/opensearch/send-message-sample.json

diff --git a/docker-compose/opensearch/README.md b/docker-compose/opensearch/README.md
new file mode 100644
index 0000000..90d0ded
--- /dev/null
+++ b/docker-compose/opensearch/README.md
@@ -0,0 +1,120 @@
+# OpenSearch Docker Compose
+
+- https://opensearch.org/docs/latest/
+- https://opensearch.org/downloads.html
+- https://github.com/opensearch-project/OpenSearch
+- https://github.com/opensearch-project/OpenSearch-Dashboards
+- https://opensearch.org/docs/latest/clients/logstash/index/
+
+## docker-compose.yml 확인
+
+elasticsearch 를 대체하는 opensearch, kibana 를 대체하는 opensearch-dashboards 그리고, logstash-oss-with-opensearch-output-plugin 의 도커 파일들을 하나로 묶어준다.
+
+- OpenSearch: Data store and search engine
+- OpenSearch Dashboards: Search frontend and visualizations
+- logstash-oss-with-opensearch-output-plugin: real-time event processing engine
+
+## 구동 (OpenSearch, OpenSearch Dashboards, Logstash)
+
+```
+docker-compose up -d
+```
+
+## Logstash 확인
+
+logstash 폴더의 config/logstash.yml 과 pipeline/logstash.conf 환경설정으로 구동된다.
+logstash의 pipeline은 input, filter, output으로 구성된다.
+
+- input: 마이크로서비스들로부터 log 이벤트들을 받는다
+- filter: input으로 받은 이벤트들을 output으로 전송하기 전에 원하는 형태로 변형한다.
+- output: OpenSearch 로 filtered 이벤트들을 전송한다.
+
+logstash.conf 파일에 다음과 같이 input과 output pipeline을 정의한다.
+tcp 5001 로 json 요청을 listen 하고 있다가, 이벤트를 수신하면 stdout으로 그대로 출력함과 동시에, opensearch로 "logstash-logs-%{+YYYY.MM.dd}" 를 인덱스 패턴으로 하여 전송한다.
+
+```
+# logstash.conf 파일
+
+input {
+  tcp {
+    port => 5001
+    codec => json
+  }
+}
+
+output {
+
+  stdout {}
+
+  opensearch {
+    hosts => ["https://opensearch:9200"]
+    index => "logstash-logs-%{+YYYY.MM.dd}"
+    user => "admin"
+    password => "admin"
+    ssl => true
+    ssl_certificate_verification => false
+  }
+}
+```
+
+tcp 5001 포트가 listen 하고 있는지 확인한다.
+
+```
+# 연결상태 확인
+nc -zv localhost 5001
+
+# 포트 확인
+lsof -i :5001
+netstat -anv | grep 5001
+```
+
+5001 로 샘플로 작성된 json 파일을 전송해 본다.
+
+```
+nc localhost 5001 < send-message-sample.json
+```
+
+최종 30 라인을 실시간 로그를 확인할 수 있다.
+
+```
+docker logs --tail 30 -f logstash
+
+[2022-02-01T18:29:15,127][WARN ][deprecation.logstash.codecs.jsonlines][main][00782acb5a7f7800ad3abe3f3cdb9bd203d3700b30ede21cdfbfeb1247906e9a] Relying on default value of `pipeline.ecs_compatibility`, which may change in a future major release of Logstash. To avoid unexpected changes when upgrading Logstash, please explicitly declare your desired ECS Compatibility mode.
+{
+      "@version" => "1",
+          "name" => "John",
+    "@timestamp" => 2022-02-01T09:29:15.137Z,
+          "port" => 59338,
+           "age" => 30,
+           "car" => nil,
+          "host" => "gateway"
+}
+```
+
+## OpenSearch Dashboards 확인
+
+브라우저에서 다음 URL로 이동하여 admin / admin 으로 접속한다.
+
+```
+http://localhost:5601/
+```
+
+- OpenSearch Dashboards > Discover 에서 확인할 수 있다.
+
+## logstash.conf 변경 적용
+
+logstash.conf를 변경한 후 적용하려면
+
+```
+docker stop logstash
+docker start logstash
+
+# 또는 restart
+docker restart logstash
+```
+
+## logstash 컨테이너 터미널 접속
+
+```
+docker exec -it logstash /bin/bash 
+```
\ No newline at end of file
diff --git a/docker-compose/opensearch/docker-compose.yml b/docker-compose/opensearch/docker-compose.yml
new file mode 100644
index 0000000..5e9de7f
--- /dev/null
+++ b/docker-compose/opensearch/docker-compose.yml
@@ -0,0 +1,63 @@
+version: '3'
+services:
+  opensearch:
+    image: opensearchproject/opensearch:latest
+    container_name: opensearch
+    environment:
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch
+      - discovery.seed_hosts=opensearch
+      - cluster.initial_master_nodes=opensearch
+      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
+        hard: 65536
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+    ports:
+      - 9200:9200
+      - 9600:9600 # required for Performance Analyzer
+    networks:
+      - egov-network
+  logstash:
+    container_name: logstash
+    image: opensearchproject/logstash-oss-with-opensearch-output-plugin:latest
+    restart: always
+    ports:
+      - "5001:5001"
+    environment:
+      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+      TZ: Asia/Seoul
+    volumes:
+      - type: bind
+        source: ${PWD}/logstash/config/logstash.yml
+        target: /usr/share/logstash/config/logstash.yml
+        read_only: true
+      - type: bind
+        source: ${PWD}/logstash/pipeline
+        target: /usr/share/logstash/pipeline
+        read_only: true
+    networks:
+      - egov-network
+  opensearch-dashboards:
+    image: opensearchproject/opensearch-dashboards:latest
+    container_name: opensearch-dashboards
+    ports:
+      - 5601:5601
+    expose:
+      - "5601"
+    environment:
+      OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
+    networks:
+      - egov-network
+
+volumes:
+  opensearch-data:
+
+networks:
+  egov-network:
diff --git a/docker-compose/opensearch/logstash/config/logstash.yml b/docker-compose/opensearch/logstash/config/logstash.yml
new file mode 100644
index 0000000..fd782d5
--- /dev/null
+++ b/docker-compose/opensearch/logstash/config/logstash.yml
@@ -0,0 +1,7 @@
+http.host: "0.0.0.0"
+
+## X-Pack security credentials
+#
+# xpack.monitoring.enabled: true
+# xpack.monitoring.elasticsearch.username: elastic
+# xpack.monitoring.elasticsearch.password: changeme
\ No newline at end of file
diff --git a/docker-compose/opensearch/logstash/pipeline/logstash.conf b/docker-compose/opensearch/logstash/pipeline/logstash.conf
new file mode 100644
index 0000000..8882d43
--- /dev/null
+++ b/docker-compose/opensearch/logstash/pipeline/logstash.conf
@@ -0,0 +1,21 @@
+# LogstashTcpSocketAppender
+input {
+  tcp {
+    port => 5001
+    codec => json
+  }
+}
+
+output {
+
+  stdout {}
+
+  opensearch {
+    hosts => ["https://opensearch:9200"]
+    index => "logstash-logs-%{+YYYY.MM.dd}"
+    user => "admin"
+    password => "admin"
+    ssl => true
+    ssl_certificate_verification => false
+  }
+}
diff --git a/docker-compose/opensearch/send-message-sample.json b/docker-compose/opensearch/send-message-sample.json
new file mode 100644
index 0000000..978d70e
--- /dev/null
+++ b/docker-compose/opensearch/send-message-sample.json
@@ -0,0 +1 @@
+{"name":"John", "age":30, "car":null}
\ No newline at end of file