forked from beep-projects/SuricataPi
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsecondrun.sh
253 lines (226 loc) · 11 KB
/
secondrun.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
#!/bin/bash
#
# Copyright (c) 2021, The beep-projects contributors
# this file originated from https://github.com/beep-projects
# Do not remove the lines above.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see https://www.gnu.org/licenses/
#
# This file is inspired by the firstrun.sh, generated by the Raspberry Pi Imager https://www.raspberrypi.org/software/
#
# This file will be called after the network has been configured by firstrun.sh
# It updates the system, installs suricata, elasticsearch, logstash and kibana,
# This script downloads and configures a lot of stuff, so it will take a while to run
# For a full description see https://github.com/beep-projects/SuricataPi/readme.md
#
#######################################
# Checks if any user is holding one of the various lock files used by apt
# and waits until they become available.
# Warning, you might get stuck forever in here
# Globals:
# None
# Arguments:
# None
# Outputs:
# None
#######################################
function waitForApt() {
while sudo fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
echo ["$(date +%T)"] waiting for access to apt lock files ...
sleep 1
done
}
#######################################
# Checks if internet can be accessed
# and waits until it becomes available.
# Warning, you might get stuck forever in here
# Globals:
# None
# Arguments:
# None
# Outputs:
# None
#######################################
function waitForInternet() {
until curl --output /dev/null --silent --head --fail http://www.google.com; do
echo ["$(date +%T)"] waiting for internet access ...
sleep 1
done
echo "done"
}
# redirect output to 'secondrun.log':
exec 3>&1 4>&2
trap 'exec 2>&4 1>&3' 0 1 2 3
exec 1>/boot/secondrun.log 2>&1
echo "START secondrun.sh"
# the following variables should be set by firstrun.sh
HOME_NET=COPY_HOME_NET_HERE
USE_LATEST_ELK=COPY_USE_LATEST_ELK_HERE
ELK_REPO_VERSION=COPY_ELK_REPO_VERSION_HERE
# update the system and install needed packages
echo "updating the system"
waitForApt
sudo apt update
waitForApt
sudo apt full-upgrade -y
# fail2ban is added to add some security to the system. Remove it, if you don't like it
waitForApt
echo "sudo apt install -y fail2ban"
sudo apt install -y fail2ban
# increase swap file size
sudo dphys-swapfile swapoff
sudo sed -i "s/^CONF_SWAPSIZE=.*/CONF_SWAPSIZE=2048/" /etc/dphys-swapfile
sudo dphys-swapfile setup
sudo dphys-swapfile swapon
# just follow the guide on https://suricata.readthedocs.io/en/latest/install.html to install suricata
# install suricata all dependencies
echo "installing suricata and all dependencies"
waitForApt
sudo apt install -y suricata
#make sure suricata is not running
sudo systemctl stop suricata
# update rules
sudo suricata-update
# add a cron job to update the rules daily
#write out current crontab
# shellcheck disable=SC2024 # we want to run crontab for sudo
sudo crontab -l > tmp_crontab
#echo new cronjob into cron file
echo "3 3 * * * suricata-update" >> tmp_crontab
#install new cron file
sudo crontab tmp_crontab
sudo rm tmp_crontab
#configure suricata
#set HOME_NET. Note we use | as delimiter for sed, because HOME_NET has / for subnet notation inside
sudo sed -i "s|^\([^#]*\)HOME_NET:.*|\1HOME_NET: \"${HOME_NET}\"|" /etc/suricata/suricata.yaml
#adjust ring buffer settings for better capture performance
#sudo sed -i "s|^\([^#]*\)#*use-mmap.*|\1use-mmap: yes|" /etc/suricata/suricata.yaml
#sudo sed -i "s|^\([^#]*\)#*mmap-locked.*|\1mmap-locked: yes|" /etc/suricata/suricata.yaml
sudo sed -i "s|^\([^#]*\)#*ring-size.*|\1ring-size: 30000|" /etc/suricata/suricata.yaml
# set the default-rule-path to /var/lib/suricata/rules/suricata.rules
sudo sed -i "s|^default-rule-path:.*|default-rule-path: /var/lib/suricata/rules/|" /etc/suricata/suricata.yaml
#now enable some free sources, you can get a list of all available free sources with
#suricata-update list-sources --free
#suricata-update list-sources --enabled
#sudo suricata-update enable-source oisf/trafficid
#sudo suricata-update enable-source sslbl/ja3-fingerprints
sudo suricata-update enable-source et/open
sudo suricata-update enable-source ptresearch/attackdetection
#sudo suricata-update enable-source sslbl/ssl-fp-blacklist
#sudo suricata-update enable-source tgreen/hunting
sudo suricata-update enable-source etnetera/aggressive
# to save space, rotate suricata logs daily and keep only one copy
sudo sed -i "s/^\(.*\)rotate .*/\1rotate 1\n\1daily/g" /etc/logrotate.d/suricata
# start suricata
sudo systemctl start suricata.service
# Setup ELK stack
################# DEB WAY ###################
if [[ ${USE_LATEST_ELK} == true ]] ; then
wget https://www.elastic.co/downloads/elasticsearch -O elasticsearch.html
ELK_VERSION=$( sed -n 's/^.*Version: <\/strong>\([^<]*\).*$/\1/p' elasticsearch.html )
ELK_REPO_VERSION=$( echo "${ELK_VERSION}" | grep -o "^[^\.]" )
fi
# Download and install the public signing key
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
# install apt-transport-https
waitForApt
sudo apt install apt-transport-https
# add the ELK repo to apt
echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/${ELK_REPO_VERSION}.x/apt stable main" | sudo tee "/etc/apt/sources.list.d/elastic-${ELK_REPO_VERSION}.x.list"
waitForApt
sudo apt update
sudo apt install -y elasticsearch kibana logstash
# limit elasticsearch heap size
echo "-Xms256m" | sudo tee /etc/elasticsearch/jvm.options.d/suricatapi.options
echo "-Xmx256m" | sudo tee -a /etc/elasticsearch/jvm.options.d/suricatapi.options
# WARNING: You are running with Java 19. To make full use of MMapDirectory, please pass '--enable-preview' to the Java command line.
echo "--enable-preview" | sudo tee -a /etc/elasticsearch/jvm.options.d/suricatapi.options
# change discovery to single-node
sudo sed -i "s/\(# For more information, consult the discovery and cluster formation module documentation.\)/\1\n#\ndiscovery.type: single-node/" /etc/elasticsearch/elasticsearch.yml
# uncomment cluster.initial_master_nodes: if not already commented
sudo sed -i "/^[^#]*cluster.initial_master_nodes:/s/^/#/g" /etc/elasticsearch/elasticsearch.yml
# disable security features for elasticsearch
sudo sed -i "s/^xpack.security.enabled:.*/xpack.security.enabled: false/g" /etc/elasticsearch/elasticsearch.yml
#sudo sed -i "s/^xpack.security.enrollment.enabled:.*/xpack.security.enrollment.enabled: false/g" /etc/elasticsearch/elasticsearch.yml
#sudo sed -i '/^.*xpack.security.http.ssl.*/{:a;n;/^.*enabled: .*/!ba;s#^.*enabled:.*# enabled: false#}' /etc/elasticsearch/elasticsearch.yml
#sudo sed -i '/^.*xpack.security.transport.ssl.*/{:a;n;/^.*enabled: .*/!ba;s#^.*enabled:.*# enabled: false#}' /etc/elasticsearch/elasticsearch.yml
# start elasticsearch
sudo systemctl start elasticsearch.service
# to proceed we need to reset the passwords for the elastic user
cd /usr/share/elasticsearch/bin || exit
#ES_PWD=$( sudo ./elasticsearch-reset-password --auto --batch -u elastic | sed -n "s/New value: \(.*\)/\1/p" )
# stop elasticsearch again
sudo systemctl stop elasticsearch.service
# during boot, elasticsearch might need a bit more time to start on the raspberry pi
sudo sed -i "s/^.*TimeoutStartSec=.*/TimeoutStartSec=180/" /usr/lib/systemd/system/elasticsearch.service
# sometimes elasticsearch crashes and needs to be restarted
sudo sed -i "/^.*ExecStart=.*/a Restart=on-failure\nRestartSec=60s" /usr/lib/systemd/system/elasticsearch.service
#copy configs for logstash
sudo cp /boot/10-suricata.conf /etc/logstash/conf.d/
#update logstash plugins
sudo /usr/share/logstash/bin/logstash-plugin update
#configure kibana to be accessible from remote machines
sudo sed -i "s/.*server.host: .*/server.host: 0.0.0.0/g" /etc/kibana/kibana.yml
# because of th elimited ressources of the raspberry pi, we have to ensure the start order of the services
# the start order should be: elasticsearch, logstash, kibana
sudo sed -i "/^Description=logstash.*/a Requires=elasticsearch.service\nAfter=elasticsearch.service" /usr/lib/systemd/system/logstash.service
sudo sed -i "s/^After=network-online.target.*/Requires=logstash.service\nAfter=logstash.service/g" /usr/lib/systemd/system/kibana.service
# TODO what was the ES_PWD needed for?
# TODO
# limit also logstash and kibana heap size
# reload and enable the new services
sudo systemctl daemon-reload
sudo systemctl enable elasticsearch.service
sudo systemctl enable kibana.service
sudo systemctl enable logstash.service
# start the ELK stack to do some configuration
sudo systemctl start elasticsearch.service
sudo systemctl start logstash.service
sudo systemctl start kibana.service
# wait until elasticsearch has fully started up
until curl --output /dev/null --silent --head --fail http://127.0.0.0:9200; do
echo ["$(date +%T)"] waiting for elasticsearch to be available ...
sleep 10
done
# the index lifecycle policy has to be imported into elasticsearch
curl --silent -X PUT -H 'Content-Type: application/json' http://127.0.0.0:9200/_ilm/policy/suricatapi-index-policy -d@/boot/suricatapi-index-policy.json
# the index template has to be imported into elasticsearch
curl --silent -X PUT -H 'Content-Type: application/json' http://127.0.0.0:9200/_index_template/suricatapi-index-template -d@/boot/suricatapi-index-template.json
# wait until kibana has started up
until curl --output /dev/null --silent --head --fail http://127.0.0.0:5601/app/home/; do
echo ["$(date +%T)"] waiting for kibana to be available ...
sleep 10
done
echo ["$(date +%T)"] kibana is available
sleep 30
# import the SuricataPi dashboards and related objects into kibana
# kibana starts in a degraded state, so it takes some time until the import is successful
RETRY_COUNT=0
while [ $RETRY_COUNT -le 10 ] ; do
SUCCESS=$( curl --silent -X POST http://127.0.0.0:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/boot/SuricataPi.ndjson | grep -Po '(?<="success":)\w+?[^,]*' )
[[ ${SUCCESS} != "true" ]] || break
RETRY_COUNT=$(( RETRY_COUNT + 1 ))
echo ["$(date +%T)"] waiting for kibana to accept the import of saved objects ...
sleep 10
done
echo "remove autoinstalled packages"
waitForApt
echo "sudo apt -y autoremove"
sudo apt -y autoremove
echo "add run /boot/thirdrun.sh command to cmdline.txt file for next reboot"
sudo sed -i '$s|$| systemd.run=/boot/thirdrun.sh systemd.run_success_action=reboot systemd.unit=kernel-command-line.target\n|' /boot/cmdline.txt
#disable the service that started this script
sudo systemctl disable secondrun.service
echo "DONE secondrun.sh, rebooting the system"
sleep 2
sudo reboot