script.sh

   1 #!/bin/sh
   2
   3 set -efux -o pipefail
   4
   5 ## Create a temporary working folder
   6 mkdir -p "tmp/"
   7 cd "tmp/"
   8
   9
  10 ## Prepare datasets
  11 curl -L "https://urlhaus.abuse.ch/downloads/csv/" -o "urlhaus.zip"
  12 curl -L "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -o "top-1m-umbrella.zip"
  13 curl -L "https://tranco-list.eu/top-1m.csv.zip" -o "top-1m-tranco.zip"
  14
  15 cp -f "../src/exclude.txt" "."
  16
  17 ## Prepare URLhaus.csv
  18 unzip -p "urlhaus.zip" | \
  19 # Convert DOS to Unix line ending
  20 dos2unix | \
  21 tr "[:upper:]" "[:lower:]" | \
  22 # Remove comment
  23 sed "/^#/d" > "URLhaus.csv"
  24
  25 ## Parse URLs
  26 cat "URLhaus.csv" | \
  27 cut -f 6 -d '"' | \
  28 cut -f 3- -d "/" | \
  29 # Domain must have at least a 'dot'
  30 grep -F "." | \
  31 # Remove invalid protocol, see #32
  32 sed -E "s/^(ttps:\/\/|https:\/|http\/)//g" | \
  33 # Remove www.
  34 sed "s/^www\.//g" | \
  35 sort -u > "urlhaus.txt"
  36
  37 ## Parse domain and IP address only
  38 cat "urlhaus.txt" | \
  39 cut -f 1 -d "/" | \
  40 cut -f 1 -d ":" | \
  41 # Remove invalid domains, see #15
  42 grep -vF "??" | \
  43 cut -f 1 -d "?" | \
  44 sort -u > "urlhaus-domains.txt"
  45
  46 ## Parse online URLs only
  47 cat "URLhaus.csv" | \
  48 grep '"online"' | \
  49 cut -f 6 -d '"' | \
  50 cut -f 3- -d "/" | \
  51 sed "s/^www\.//g" | \
  52 sort -u > "urlhaus-online.txt"
  53
  54 cat "urlhaus-online.txt" | \
  55 cut -f 1 -d "/" | \
  56 cut -f 1 -d ":" | \
  57 grep -vF "??" | \
  58 cut -f 1 -d "?" | \
  59 sort -u > "urlhaus-domains-online.txt"
  60
  61
  62 ## Parse the Umbrella 1 Million
  63 unzip -p "top-1m-umbrella.zip" | \
  64 dos2unix | \
  65 tr "[:upper:]" "[:lower:]" | \
  66 # Parse domains only
  67 cut -f 2 -d "," | \
  68 grep -F "." | \
  69 # Remove www.
  70 sed "s/^www\.//g" | \
  71 sort -u > "top-1m-umbrella.txt"
  72
  73 ## Parse the Tranco 1 Million
  74 unzip -p "top-1m-tranco.zip" | \
  75 dos2unix | \
  76 tr "[:upper:]" "[:lower:]" | \
  77 # Parse domains only
  78 cut -f 2 -d "," | \
  79 grep -F "." | \
  80 # Remove www.
  81 sed "s/^www\.//g" | \
  82 sort -u > "top-1m-tranco.txt"
  83
  84 # Merge Umbrella and self-maintained top domains
  85 cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" | \
  86 sort -u > "top-1m-well-known.txt"
  87
  88
  89 ## Parse popular domains from URLhaus
  90 cat "urlhaus-domains.txt" | \
  91 # grep match whole line
  92 grep -Fx -f "top-1m-well-known.txt" > "urlhaus-top-domains.txt"
  93
  94
  95 ## Parse domains from URLhaus excluding popular domains
  96 cat "urlhaus-domains.txt" | \
  97 grep -F -vf "urlhaus-top-domains.txt" | \
  98 # Remove blank lines
  99 sed "/^$/d" > "malware-domains.txt"
 100
 101 cat "urlhaus-domains-online.txt" | \
 102 grep -F -vf "urlhaus-top-domains.txt" | \
 103 sed "/^$/d" > "malware-domains-online.txt"
 104
 105 ## Parse malware URLs from popular domains
 106 cat "urlhaus.txt" | \
 107 grep -F -f "urlhaus-top-domains.txt" | \
 108 sed "s/^/||/g" | \
 109 sed "s/$/\$all/g" > "malware-url-top-domains.txt"
 110
 111 cat "urlhaus-online.txt" | \
 112 grep -F -f "urlhaus-top-domains.txt" | \
 113 sed "s/^/||/g" | \
 114 sed "s/$/\$all/g" > "malware-url-top-domains-online.txt"
 115
 116 cat "urlhaus-online.txt" | \
 117 grep -F -f "urlhaus-top-domains.txt" > "malware-url-top-domains-raw-online.txt"
 118
 119
 120 ## Merge malware domains and URLs
 121 CURRENT_TIME="$(date -R -u)"
 122 FIRST_LINE="! Title: Malicious URL Blocklist"
 123 SECOND_LINE="! Updated: $CURRENT_TIME"
 124 THIRD_LINE="! Expires: 1 day (update frequency)"
 125 FOURTH_LINE="! Homepage: https://gitlab.com/curben/urlhaus-filter"
 126 FIFTH_LINE="! License: https://gitlab.com/curben/urlhaus-filter#license"
 127 SIXTH_LINE="! Source: https://urlhaus.abuse.ch/api/"
 128 COMMENT_ABP="$FIRST_LINE\n$SECOND_LINE\n$THIRD_LINE\n$FOURTH_LINE\n$FIFTH_LINE\n$SIXTH_LINE"
 129
 130
 131 cat "malware-domains.txt" "malware-url-top-domains.txt" | \
 132 sort | \
 133 sed '1 i\'"$COMMENT_ABP"'' > "../urlhaus-filter.txt"
 134
 135 cat "malware-domains-online.txt" "malware-url-top-domains-online.txt" | \
 136 sort | \
 137 sed '1 i\'"$COMMENT_ABP"'' | \
 138 sed "1s/Malicious/Online Malicious/" > "../urlhaus-filter-online.txt"
 139
 140
 141 # Adguard Home (#19, #22)
 142 cat "malware-domains.txt" | \
 143 sed "s/^/||/g" | \
 144 sed "s/$/^/g" > "malware-domains-adguard-home.txt"
 145
 146 cat "malware-domains-online.txt" | \
 147 sed "s/^/||/g" | \
 148 sed "s/$/^/g" > "malware-domains-online-adguard-home.txt"
 149
 150 cat "malware-domains-adguard-home.txt" | \
 151 sort | \
 152 sed '1 i\'"$COMMENT_ABP"'' | \
 153 sed "1s/Blocklist/Blocklist (AdGuard Home)/" > "../urlhaus-filter-agh.txt"
 154
 155 cat "malware-domains-online-adguard-home.txt" | \
 156 sort | \
 157 sed '1 i\'"$COMMENT_ABP"'' | \
 158 sed "1s/Malicious/Online Malicious/" | \
 159 sed "1s/Blocklist/Blocklist (AdGuard Home)/" > "../urlhaus-filter-agh-online.txt"
 160
 161
 162 # Adguard browser extension
 163 cat "malware-domains.txt" | \
 164 sed "s/^/||/g" | \
 165 sed "s/$/\$all/g" > "malware-domains-adguard.txt"
 166
 167 cat "malware-domains-online.txt" | \
 168 sed "s/^/||/g" | \
 169 sed "s/$/\$all/g" > "malware-domains-online-adguard.txt"
 170
 171 cat "malware-domains-adguard.txt" "malware-url-top-domains.txt" | \
 172 sort | \
 173 sed '1 i\'"$COMMENT_ABP"'' | \
 174 sed "1s/Blocklist/Blocklist (AdGuard)/" > "../urlhaus-filter-ag.txt"
 175
 176 cat "malware-domains-online-adguard.txt" "malware-url-top-domains-online.txt" | \
 177 sort | \
 178 sed '1 i\'"$COMMENT_ABP"'' | \
 179 sed "1s/Malicious/Online Malicious/" | \
 180 sed "1s/Blocklist/Blocklist (AdGuard)/" > "../urlhaus-filter-ag-online.txt"
 181
 182
 183 # Vivaldi
 184 cat "malware-domains.txt" | \
 185 sed "s/^/||/g" | \
 186 sed "s/$/\$document/g" > "malware-domains-vivaldi.txt"
 187
 188 cat "malware-domains-online.txt" | \
 189 sed "s/^/||/g" | \
 190 sed "s/$/\$document/g" > "malware-domains-online-vivaldi.txt"
 191
 192 cat "malware-domains-vivaldi.txt" "malware-url-top-domains.txt" | \
 193 sed "s/\$all$/\$document/g" | \
 194 sort | \
 195 sed '1 i\'"$COMMENT_ABP"'' | \
 196 sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../urlhaus-filter-vivaldi.txt"
 197
 198 cat "malware-domains-online-vivaldi.txt" "malware-url-top-domains-online.txt" | \
 199 sed "s/\$all$/\$document/g" | \
 200 sort | \
 201 sed '1 i\'"$COMMENT_ABP"'' | \
 202 sed "1s/Malicious/Online Malicious/" | \
 203 sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../urlhaus-filter-vivaldi-online.txt"
 204
 205
 206 ## Domains-only blocklist
 207 # awk + head is a workaround for sed prepend
 208 COMMENT=$(printf "$COMMENT_ABP" | sed "s/^!/#/g" | sed "1s/URL/Domains/" | awk '{printf "%s\\n", $0}' | head -c -2)
 209 COMMENT_ONLINE=$(printf "$COMMENT" | sed "1s/Malicious/Online Malicious/" | awk '{printf "%s\\n", $0}' | head -c -2)
 210
 211 cat "malware-domains.txt" | \
 212 sort | \
 213 sed '1 i\'"$COMMENT"'' > "../urlhaus-filter-domains.txt"
 214
 215 cat "malware-domains-online.txt" | \
 216 sort | \
 217 sed '1 i\'"$COMMENT_ONLINE"'' > "../urlhaus-filter-domains-online.txt"
 218
 219
 220 ## Hosts only
 221 cat "malware-domains.txt" | \
 222 sort | \
 223 # Remove IPv4 address
 224 grep -vE "^([0-9]{1,3}[\.]){3}[0-9]{1,3}$" > "malware-hosts.txt"
 225
 226 cat "malware-domains-online.txt" | \
 227 sort | \
 228 # Remove IPv4 address
 229 grep -vE "^([0-9]{1,3}[\.]){3}[0-9]{1,3}$" > "malware-hosts-online.txt"
 230
 231
 232 ## Hosts file blocklist
 233 cat "malware-hosts.txt" | \
 234 sed "s/^/0.0.0.0 /g" | \
 235 # Re-insert comment
 236 sed '1 i\'"$COMMENT"'' | \
 237 sed "1s/Domains/Hosts/" > "../urlhaus-filter-hosts.txt"
 238
 239 cat "malware-hosts-online.txt" | \
 240 sed "s/^/0.0.0.0 /g" | \
 241 sed '1 i\'"$COMMENT_ONLINE"'' | \
 242 sed "1s/Domains/Hosts/" > "../urlhaus-filter-hosts-online.txt"
 243
 244
 245 ## Dnsmasq-compatible blocklist
 246 cat "malware-hosts.txt" | \
 247 sed "s/^/address=\//g" | \
 248 sed "s/$/\/0.0.0.0/g" | \
 249 sed '1 i\'"$COMMENT"'' | \
 250 sed "1s/Blocklist/dnsmasq Blocklist/" > "../urlhaus-filter-dnsmasq.conf"
 251
 252 cat "malware-hosts-online.txt" | \
 253 sed "s/^/address=\//g" | \
 254 sed "s/$/\/0.0.0.0/g" | \
 255 sed '1 i\'"$COMMENT_ONLINE"'' | \
 256 sed "1s/Blocklist/dnsmasq Blocklist/" > "../urlhaus-filter-dnsmasq-online.conf"
 257
 258
 259 ## BIND-compatible blocklist
 260 cat "malware-hosts.txt" | \
 261 sed 's/^/zone "/g' | \
 262 sed 's/$/" { type master; notify no; file "null.zone.file"; };/g' | \
 263 sed '1 i\'"$COMMENT"'' | \
 264 sed "1s/Blocklist/BIND Blocklist/" > "../urlhaus-filter-bind.conf"
 265
 266 cat "malware-hosts-online.txt" | \
 267 sed 's/^/zone "/g' | \
 268 sed 's/$/" { type master; notify no; file "null.zone.file"; };/g' | \
 269 sed '1 i\'"$COMMENT_ONLINE"'' | \
 270 sed "1s/Blocklist/BIND Blocklist/" > "../urlhaus-filter-bind-online.conf"
 271
 272
 273 ## DNS Response Policy Zone (RPZ)
 274 CURRENT_UNIX_TIME="$(date +%s)"
 275 RPZ_SYNTAX="\n\$TTL 30\n@ IN SOA rpz.curben.gitlab.io. hostmaster.rpz.curben.gitlab.io. $CURRENT_UNIX_TIME 86400 3600 604800 30\n NS localhost.\n"
 276
 277 cat "malware-hosts.txt" | \
 278 sed "s/$/ CNAME ./g" | \
 279 sed '1 i\'"$RPZ_SYNTAX"'' | \
 280 sed '1 i\'"$COMMENT"'' | \
 281 sed "s/^#/;/g" | \
 282 sed "1s/Blocklist/RPZ Blocklist/" > "../urlhaus-filter-rpz.conf"
 283
 284 cat "malware-hosts-online.txt" | \
 285 sed "s/$/ CNAME ./g" | \
 286 sed '1 i\'"$RPZ_SYNTAX"'' | \
 287 sed '1 i\'"$COMMENT_ONLINE"'' | \
 288 sed "s/^#/;/g" | \
 289 sed "1s/Blocklist/RPZ Blocklist/" > "../urlhaus-filter-rpz-online.conf"
 290
 291
 292 ## Unbound-compatible blocklist
 293 cat "malware-hosts.txt" | \
 294 sed 's/^/local-zone: "/g' | \
 295 sed 's/$/" always_nxdomain/g' | \
 296 sed '1 i\'"$COMMENT"'' | \
 297 sed "1s/Blocklist/Unbound Blocklist/" > "../urlhaus-filter-unbound.conf"
 298
 299 cat "malware-hosts-online.txt" | \
 300 sed 's/^/local-zone: "/g' | \
 301 sed 's/$/" always_nxdomain/g' | \
 302 sed '1 i\'"$COMMENT_ONLINE"'' | \
 303 sed "1s/Blocklist/Unbound Blocklist/" > "../urlhaus-filter-unbound-online.conf"
 304
 305
 306 ## dnscrypt-proxy blocklists
 307 # name-based
 308 cat "malware-hosts.txt" | \
 309 sed '1 i\'"$COMMENT"'' | \
 310 sed "1s/Domains/Names/" > "../urlhaus-filter-dnscrypt-blocked-names.txt"
 311
 312 cat "malware-hosts-online.txt" | \
 313 sed '1 i\'"$COMMENT_ONLINE"'' | \
 314 sed "1s/Domains/Names/" > "../urlhaus-filter-dnscrypt-blocked-names-online.txt"
 315
 316 ## IPv4-based
 317 cat "malware-domains.txt" | \
 318 sort | \
 319 grep -E "^([0-9]{1,3}[\.]){3}[0-9]{1,3}$" | \
 320 sed '1 i\'"$COMMENT"'' | \
 321 sed "1s/Domains/IPs/" > "../urlhaus-filter-dnscrypt-blocked-ips.txt"
 322
 323 cat "malware-domains-online.txt" | \
 324 sort | \
 325 grep -E "^([0-9]{1,3}[\.]){3}[0-9]{1,3}$" | \
 326 sed '1 i\'"$COMMENT_ONLINE"'' | \
 327 sed "1s/Domains/IPs/" > "../urlhaus-filter-dnscrypt-blocked-ips-online.txt"
 328
 329
 330 ## Temporarily disable command print
 331 set +x
 332
 333
 334 # Snort & Suricata
 335 rm -f "../urlhaus-filter-snort2-online.rules" \
 336   "../urlhaus-filter-snort3-online.rules" \
 337   "../urlhaus-filter-suricata-online.rules"
 338
 339 SID="100000001"
 340 while read DOMAIN; do
 341   SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$DOMAIN\"; content:\"Host\"; http_header; classtype:trojan-activity; sid:$SID; rev:1;)"
 342
 343   SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; http_header:field host; content:\"$DOMAIN\",nocase; classtype:trojan-activity; sid:$SID; rev:1;)"
 344
 345   SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.host; content:\"$DOMAIN\"; classtype:trojan-activity; sid:$SID; rev:1;)"
 346
 347   echo "$SN_RULE" >> "../urlhaus-filter-snort2-online.rules"
 348   echo "$SN3_RULE" >> "../urlhaus-filter-snort3-online.rules"
 349   echo "$SR_RULE" >> "../urlhaus-filter-suricata-online.rules"
 350
 351   SID=$(( $SID + 1 ))
 352 done < "malware-domains-online.txt"
 353
 354 while read URL; do
 355   HOST=$(echo "$URL" | cut -d"/" -f1)
 356   URI=$(echo "$URL" | sed -e "s/^$HOST//" -e "s/;/\\\;/g")
 357
 358   # Snort2 only supports <=2047 characters of `content`
 359   SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$(echo $URI | cut -c -2047)\"; http_uri; nocase; content:\"$HOST\"; content:\"Host\"; http_header; classtype:trojan-activity; sid:$SID; rev:1;)"
 360
 361   SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; http_header:field host; content:\"$HOST\",nocase; http_uri; content:\"$URI\",nocase; classtype:trojan-activity; sid:$SID; rev:1;)"
 362
 363   SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.uri; content:\"$URI\"; endswith; nocase; http.host; content:\"$HOST\"; classtype:trojan-activity; sid:$SID; rev:1;)"
 364
 365   echo "$SN_RULE" >> "../urlhaus-filter-snort2-online.rules"
 366   echo "$SN3_RULE" >> "../urlhaus-filter-snort3-online.rules"
 367   echo "$SR_RULE" >> "../urlhaus-filter-suricata-online.rules"
 368
 369   SID=$(( $SID + 1 ))
 370 done < "malware-url-top-domains-raw-online.txt"
 371
 372 ## Re-enable command print
 373 set -x
 374
 375 sed -i '1 i\'"$COMMENT_ONLINE"'' "../urlhaus-filter-snort2-online.rules"
 376 sed -i "1s/Domains Blocklist/URL Snort2 Ruleset/" "../urlhaus-filter-snort2-online.rules"
 377
 378 sed -i '1 i\'"$COMMENT_ONLINE"'' "../urlhaus-filter-snort3-online.rules"
 379 sed -i "1s/Domains Blocklist/URL Snort3 Ruleset/" "../urlhaus-filter-snort3-online.rules"
 380
 381 sed -i '1 i\'"$COMMENT_ONLINE"'' "../urlhaus-filter-suricata-online.rules"
 382 sed -i "1s/Domains Blocklist/URL Suricata Ruleset/" "../urlhaus-filter-suricata-online.rules"
 383
 384
 385 ## IE blocklist
 386 COMMENT_IE="msFilterList\n$COMMENT\n: Expires=1\n#"
 387 COMMENT_ONLINE_IE="msFilterList\n$COMMENT_ONLINE\n: Expires=1\n#"
 388
 389 cat "malware-hosts.txt" | \
 390 sed "s/^/-d /g" | \
 391 sed '1 i\'"$COMMENT_IE"'' | \
 392 sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../urlhaus-filter.tpl"
 393
 394 cat "malware-hosts-online.txt" | \
 395 sed "s/^/-d /g" | \
 396 sed '1 i\'"$COMMENT_ONLINE_IE"'' | \
 397 sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../urlhaus-filter-online.tpl"
 398
 399
 400 ## Clean up artifacts
 401 rm "URLhaus.csv" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt"
 402
 403
 404 cd ../