From cb12fb48ae1591cb418bec7eeb80b41410c63f2a Mon Sep 17 00:00:00 2001 From: Hariom Chaturvedi Date: Sat, 3 Jul 2021 03:02:34 +0530 Subject: [PATCH 1/2] Added evtxtoelk in docker-compose Signed-off-by: Hariom Chaturvedi --- docker-compose-evtxtoelk.yaml | 19 ------------------- docker-compose.yaml | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 19 deletions(-) delete mode 100644 docker-compose-evtxtoelk.yaml diff --git a/docker-compose-evtxtoelk.yaml b/docker-compose-evtxtoelk.yaml deleted file mode 100644 index 43b61bc..0000000 --- a/docker-compose-evtxtoelk.yaml +++ /dev/null @@ -1,19 +0,0 @@ -version: '3.2' - -services: - evtxtoelk: - container_name: pcapmonkey_evtxtoelk - image: certego/evtxtoelk:v1.1.0 - entrypoint: - - python - - evtxtoelk.py - - /var/log/event_logs - - elasticsearch:9200 - - -s - - "2000" - - -i - - windows_events_original_ts - depends_on: - - elasticsearch - volumes: - - ./import_event_logs:/var/log/event_logs:ro \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index e42a0ea..74ca42e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -102,6 +102,23 @@ services: - ./logs/zeek:/var/log/zeek - ./logs/zeek/extracted_files:/opt/zeek/extracted + evtxtoelk: + container_name: pcapmonkey_evtxtoelk + image: certego/evtxtoelk:v1.1.0 + entrypoint: + - python + - evtxtoelk.py + - /var/log/event_logs + - elasticsearch:9200 + - -s + - "2000" + - -i + - windows_events_original_ts + depends_on: + - elasticsearch + volumes: + - ./import_event_logs:/var/log/event_logs:ro + volumes: elasticsearch_data: From dd240da946696bdddd690fa3ce09db3cbe810326 Mon Sep 17 00:00:00 2001 From: Hariom Chaturvedi Date: Tue, 6 Jul 2021 02:32:50 +0530 Subject: [PATCH 2/2] Added filebeat.ndjson and update kibana.ndjson Signed-off-by: Hariom Chaturvedi --- README.md | 20 ++++++++++---------- docker-compose.yaml | 2 +- filebeat.ndjson | 10 ++++++++++ 3 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 filebeat.ndjson diff --git a/README.md b/README.md index 3f50c0a..7848ed0 100644 --- a/README.md +++ b/README.md @@ -163,20 +163,20 @@ Now you can find the Event logs in `windows_events_original_ts` and in `windows_ If you would like like to use ECS (elastic common schema) to process your Zeek and Suricata logs you should launch also ``filebeat`` Start the Elasticsearch with filebeat container: -``` +```bash sudo docker-compose up -d elasticsearch filebeat kibana ``` -Create a new index for filebeat. +Import the `filebeat.ndjson` objects to Kibana. -```Stack Management -> Index Patterns -> Create new pattern.``` +```Stack Management -> Saved Objects -> Import``` -![Kibana Index Patterns](./images/kibana_management_filebeat.png) +The `filebeat.ndjson` objects are: -Enter index pattern name `filebeat-*` and select `@timestamp` as the time field. +| Object Name | Description | +|-----------------------------|----------------------------------------------------| +| filebeat-* | @timestamp indexed Filebeat Patterns. | +| file* | Filebeat Patterns indexed with log ingestion time. | +| winlogbeat* | Windows Event Log Pattern indexed with ingestion time. | -![New Index Pattern](./images/kibana_new_index_filebeat.png) - -You can view your logs in `filebeat-*` index in the Discover section. - -![Filebeat Index Pattern](./images/kibana_index_dropdown_filebeat.png) +You can now analyze the data in the Discover section. \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 74ca42e..1c04aae 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -113,7 +113,7 @@ services: - -s - "2000" - -i - - windows_events_original_ts + - winlogbeat depends_on: - elasticsearch volumes: diff --git a/filebeat.ndjson b/filebeat.ndjson new file mode 100644 index 0000000..e7b46b3 --- /dev/null +++ b/filebeat.ndjson @@ -0,0 +1,10 @@ +{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"filebeat-*"},"coreMigrationVersion":"7.13.2","id":"5b4ca870-db71-11eb-8d71-e148878ab61e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1625517347550,38],"type":"index-pattern","updated_at":"2021-07-05T20:35:47.550Z","version":"WzE5LDFd"} +{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"event.ingested","title":"file*"},"coreMigrationVersion":"7.13.2","id":"6e2380e0-db71-11eb-8d71-e148878ab61e","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1625517347550,17],"type":"index-pattern","updated_at":"2021-07-05T20:35:47.550Z","version":"WzIwLDFd"} +{"attributes":{"buildNum":40943,"defaultIndex":"5b4ca870-db71-11eb-8d71-e148878ab61e"},"coreMigrationVersion":"7.13.2","id":"7.13.2","migrationVersion":{"config":"7.13.0"},"references":[],"sort":[1625517370611,33],"type":"config","updated_at":"2021-07-05T20:36:10.611Z","version":"WzMwLDFd"} +{"attributes":{"columns":[],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"tags:\\\"zeek.dns\\\" \",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"Zeek DNS Logs","version":1},"coreMigrationVersion":"7.13.2","id":"72cdc7b0-db74-11eb-8d71-e148878ab61e","migrationVersion":{"search":"7.9.3"},"references":[{"id":"5b4ca870-db71-11eb-8d71-e148878ab61e","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1625517347550,20],"type":"search","updated_at":"2021-07-05T20:35:47.550Z","version":"WzIyLDFd"} +{"attributes":{"columns":[],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"tags:\\\"zeek.ssl\\\" \",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"Zeek SSL Logs","version":1},"coreMigrationVersion":"7.13.2","id":"7b1185c0-db73-11eb-8d71-e148878ab61e","migrationVersion":{"search":"7.9.3"},"references":[{"id":"5b4ca870-db71-11eb-8d71-e148878ab61e","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1625517347550,22],"type":"search","updated_at":"2021-07-05T20:35:47.550Z","version":"WzIzLDFd"} +{"attributes":{"fieldAttrs":"{}","fields":"[]","runtimeFieldMap":"{}","timeFieldName":"@timestamp","title":"winlogbeat*"},"coreMigrationVersion":"7.13.2","id":"a6de76c0-ddd2-11eb-a3fd-31e52f3ab3dc","migrationVersion":{"index-pattern":"7.11.0"},"references":[],"sort":[1625518237740,245],"type":"index-pattern","updated_at":"2021-07-05T20:50:37.740Z","version":"WzI0MiwxXQ=="} +{"attributes":{"columns":[],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"suricata.eve.event_type : \\\"alert\\\" \",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"Suricata Alert Logs","version":1},"coreMigrationVersion":"7.13.2","id":"a960aa40-ddd1-11eb-a3fd-31e52f3ab3dc","migrationVersion":{"search":"7.9.3"},"references":[{"id":"6e2380e0-db71-11eb-8d71-e148878ab61e","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1625517812452,158],"type":"search","updated_at":"2021-07-05T20:43:32.452Z","version":"WzE2MSwxXQ=="} +{"attributes":{"columns":[],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"tags:\\\"zeek.http\\\" \",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"Zeek HTTP Logs","version":1},"coreMigrationVersion":"7.13.2","id":"c9effa20-db7b-11eb-8d71-e148878ab61e","migrationVersion":{"search":"7.9.3"},"references":[{"id":"5b4ca870-db71-11eb-8d71-e148878ab61e","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1625517347550,25],"type":"search","updated_at":"2021-07-05T20:35:47.550Z","version":"WzI1LDFd"} +{"attributes":{"columns":[],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"tags:\\\"zeek.files\\\" \",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[["@timestamp","desc"]],"title":"Zeek File Logs","version":1},"coreMigrationVersion":"7.13.2","id":"fd820ce0-db74-11eb-8d71-e148878ab61e","migrationVersion":{"search":"7.9.3"},"references":[{"id":"5b4ca870-db71-11eb-8d71-e148878ab61e","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"sort":[1625517347550,29],"type":"search","updated_at":"2021-07-05T20:35:47.550Z","version":"WzI3LDFd"} +{"exportedCount":9,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file