#!/bin/bash
#
# SCRIPT: MakeFeed.sh
# AUTHOR: David Waugh RSA Technical Support
# DATE: 24 March 2016
# REV: 2
#
# PLATFORM: Centos 6
#
# PURPOSE:
# LICENSE: GNU Public License v2 (http://gnu.org/licenses/)
# Copyright (C) 2016
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.
#
# See the GNU General Public License for more details.
#
# Changes:
# 1.2 adds &expirey=600 to prevent queries giving a 408 TimeOut error Message
# adds the ability for up to 5 different values of a metakey. We will create a seperate feed for each set of values
# For example output myfeed1.csv will contain the first set of values. myfeed2.csv will contain the second set of values as so on.
#
###################################################################################
##### PARAMETERS THAT CAN BE CHANGED
MAXRESULTS=200
USER=admin
PASSWORD=netwitness
BROKER=192.168.123.249:50103
OUTPUTFEEDFILE=/var/www/html/myfeed
OUTPUTFEEDFILEEXT=.csv
MAXVALUESPERKEY=5
# The Callback key we will use in our feed
LINKFIELD="ip.src"
#The Meta Keys that we want in our feed
METAKEYS="os.name os.platform os.sname os.type os.ver brand model bot.info clientinfo.type clientinfo.name clientinfo.sname clientinfo.ver clientinfo.plat"
###### END PARAMETERS THAT CAN BE CHANGED
rawurlencode() {
local string="${1}"
local strlen=${#string}
local encoded=""
local pos c o
for (( pos=0 ; pos<strlen ; pos++ )); do
c=${string:$pos:1}
case "$c" in
[-_=.:/\&\?~a-zA-Z0-9] ) o="${c}" ;;
* ) printf -v o '%%%02x' "'$c"
esac
encoded+="${o}"
done
echo "${encoded}" # You can either set a return variable (FASTER)
REPLY="${encoded}" #+or echo the result (EASIER)... or both... :p
}
selectdistinct() {
# This function takes a string of words "the lazy brown fox"
# It outputs "disinct(the),distinct(lazy),distinct(brown),distinct(fox)"
OUTPUT=""
for word in ${1}
do
OUTPUT+=$(echo "distinct($word),")
done
#Remove the last comma
SELECTDISTINCT=$(echo "${OUTPUT%?}")
}
blankresult() {
#This function takes a string of words "the lazy brown fox"
# It output "^^^" basically a ^ for each space
OUTPUT=""
for word in ${1}
do
OUTPUT+=$(echo "\^")
done
BLANKRESULT=$OUTPUT
}
CurrentTime=$(date -u +"%Y-%b-%d %H:%M:%S")
#echo "Current Time is: " $CurrentTime
PreviousTime=$(date -u -d '1 hour ago' +"%Y-%b-%d %H:%M:%S")
#echo "1 Hour Ago was: " $PreviousTime
EXISTS=$(echo "$METAKEYS" |sed 's/ / exists ||/g')
SELECT=$(echo "$METAKEYS" |sed 's/ /,/g')
selectdistinct "$METAKEYS"
EXISTS="$EXISTS exists"
#echo "EXISTS: " $EXISTS
#echo "SELECT: " $SELECT
#echo "SELECTDISTINCT:" $SELECTDISTINCT
blankresult "$METAKEYS"
#echo "BLANKRESULT: " $BLANKRESULT
# First Get All Link Fields where our Meta Keys Exist
#URL="http://$BROKER/sdk?msg=query&expirey=600&size=$MAXRESULTS&query=select distinct($LINKFIELD) where $EXISTS&time='$PreviousTime'-'$CurrentTime'&force-content-type=text/plain"
URL="http://$BROKER/sdk?msg=values&expirey=600&size=$MAXRESULTS&fieldName=$LINKFIELD&where=$EXISTS&force-content-type=text/plain"
#echo "URL: " $URL
URL=$( rawurlencode "$URL")
#echo "Encoded URL: " $URL
curl -s --user "$USER:$PASSWORD" "$URL" |grep value |cut -d " " -f 9 |cut -d "=" -f 2 >/tmp/linkfields.txt
#curl -s --user "$USER:$PASSWORD" "$URL" >/tmp/linkfields.output
# Print out the header for our output feed files
COUNTER=1
while [ $COUNTER -le $MAXVALUESPERKEY ]; do
echo "#"$LINKFIELD,$SELECT > $OUTPUTFEEDFILE$COUNTER$OUTPUTFEEDFILEEXT
let COUNTER=COUNTER+1
done
while read link; do
# We know that our linkfield has values for out METAKEYS so lets get the values over the last hour
URL="http://$BROKER/sdk?msg=query&expirey=600&size=$MAXRESULTS&query=select $SELECTDISTINCT where $LINKFIELD=$link&time='$PreviousTime'-'$CurrentTime'&force-content-type=text/plain"
URL=$( rawurlencode "$URL")
# Our Result will contain ^M Line endings so we replace these with newline characters
(curl -s --user "$USER:$PASSWORD" "$URL" >/tmp/$link.result.txt)
# We shouldnt, but sometimes we are seeing 408 Request Timeout so resubmit query if we do
while grep -Fxq "408 Request Timeout: Request Timeout" /tmp/$link.result.txt;
do
#echo "408 Request Timeout: Request Timeout FOUND: " $link " Will retry "
(curl -s --user "$USER:$PASSWORD" "$URL" >/tmp/$link.result.txt)
done
dos2unix -q -o /tmp/$link.result.txt
COUNTER=1
while [ $COUNTER -le $MAXVALUESPERKEY ]; do
OUTPUT=$link
OUTPUT+="^"
for meta in $METAKEYS
do
#cat /tmp/$link.result.txt |grep value |awk -v var=$meta -v count=$COUNTER '$0 ~ var {i++}i==count{print; exit}' >/tmp/$link.$meta.$COUNTER.output.txt
OUTPUT+=$(cat /tmp/$link.result.txt |grep value |awk -v var=$meta -v count=$COUNTER '$0 ~ var {i++}i==count{print; exit}' |cut -d "=" -f 6 |sed 's/ type//' |xargs )
OUTPUT+="^"
done
echo $OUTPUT |grep -v "$BLANKRESULT" >> $OUTPUTFEEDFILE$COUNTER$OUTPUTFEEDFILEEXT
let COUNTER=COUNTER+1
done
rm -rf /tmp/$link.result.txt
done </tmp/linkfields.txt
