#!/bin/bash
#
# SCRIPT: MakeFeed.sh
# AUTHOR: David Waugh RSA Technical Support
# DATE: 24 March 2016
# REV: 2
#
# PLATFORM: Centos 6 
#
# PURPOSE: 
# LICENSE: GNU Public License v2 (http://gnu.org/licenses/)
# Copyright (C) 2016 
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.
#
# See the GNU General Public License for more details.
#
# Changes: 
# 1.2 	adds &expirey=600 to prevent queries giving a 408 TimeOut error Message
# 	adds the ability for up to 5 different values of a metakey. We will create a seperate feed for each set of values
#	For example output myfeed1.csv will contain the first set of values. myfeed2.csv will contain the second set of values as so on.
#	
###################################################################################


##### PARAMETERS THAT CAN BE CHANGED
MAXRESULTS=200
USER=admin
PASSWORD=netwitness
BROKER=192.168.123.249:50103
OUTPUTFEEDFILE=/var/www/html/myfeed
OUTPUTFEEDFILEEXT=.csv
MAXVALUESPERKEY=5
# The Callback key we will use in our feed
LINKFIELD="ip.src"

#The Meta Keys that we want in our feed
METAKEYS="os.name os.platform os.sname os.type os.ver brand model bot.info clientinfo.type clientinfo.name clientinfo.sname clientinfo.ver clientinfo.plat"
###### END PARAMETERS THAT CAN BE CHANGED

rawurlencode() {
  local string="${1}"
  local strlen=${#string}
  local encoded=""
  local pos c o

  for (( pos=0 ; pos<strlen ; pos++ )); do
     c=${string:$pos:1}
     case "$c" in
        [-_=.:/\&\?~a-zA-Z0-9] ) o="${c}" ;;
        * )               printf -v o '%%%02x' "'$c"
     esac
     encoded+="${o}"
  done
  echo "${encoded}"    # You can either set a return variable (FASTER)
  REPLY="${encoded}"   #+or echo the result (EASIER)... or both... :p
}

selectdistinct() {
# This function takes a string of words "the lazy brown fox"
# It outputs "disinct(the),distinct(lazy),distinct(brown),distinct(fox)"
  OUTPUT=""
  for word in ${1} 
  do
   OUTPUT+=$(echo "distinct($word),")  
  done
  #Remove the last comma
  SELECTDISTINCT=$(echo "${OUTPUT%?}")
}

blankresult() {
#This function takes a string of words "the lazy brown fox"
# It output "^^^" basically a ^ for each space
OUTPUT=""
  for word in ${1}
  do
   OUTPUT+=$(echo "\^")
  done
  BLANKRESULT=$OUTPUT
}

CurrentTime=$(date -u +"%Y-%b-%d %H:%M:%S")
#echo "Current Time is: " $CurrentTime
PreviousTime=$(date -u -d '1 hour ago' +"%Y-%b-%d %H:%M:%S")
#echo "1 Hour Ago was: " $PreviousTime

EXISTS=$(echo "$METAKEYS" |sed 's/ / exists ||/g')
SELECT=$(echo "$METAKEYS" |sed 's/ /,/g')
selectdistinct "$METAKEYS"
EXISTS="$EXISTS exists"
#echo "EXISTS: " $EXISTS
#echo "SELECT: " $SELECT
#echo "SELECTDISTINCT:" $SELECTDISTINCT

blankresult  "$METAKEYS"
#echo "BLANKRESULT: " $BLANKRESULT
# First Get All Link Fields where our Meta Keys Exist
#URL="http://$BROKER/sdk?msg=query&expirey=600&size=$MAXRESULTS&query=select distinct($LINKFIELD) where $EXISTS&time='$PreviousTime'-'$CurrentTime'&force-content-type=text/plain"
URL="http://$BROKER/sdk?msg=values&expirey=600&size=$MAXRESULTS&fieldName=$LINKFIELD&where=$EXISTS&force-content-type=text/plain"

#echo "URL: " $URL
URL=$( rawurlencode "$URL")
#echo "Encoded URL: " $URL
curl -s --user "$USER:$PASSWORD" "$URL" |grep value |cut -d " " -f 9 |cut -d "=" -f 2 >/tmp/linkfields.txt
#curl -s --user "$USER:$PASSWORD" "$URL" >/tmp/linkfields.output

# Print out the header for our output feed files

COUNTER=1
while [ $COUNTER -le $MAXVALUESPERKEY ]; do
	echo "#"$LINKFIELD,$SELECT >  $OUTPUTFEEDFILE$COUNTER$OUTPUTFEEDFILEEXT
	let COUNTER=COUNTER+1
done

while read link; do
  # We know that our linkfield has values for out METAKEYS so lets get the values over the last hour
  URL="http://$BROKER/sdk?msg=query&expirey=600&size=$MAXRESULTS&query=select $SELECTDISTINCT where $LINKFIELD=$link&time='$PreviousTime'-'$CurrentTime'&force-content-type=text/plain"
  URL=$( rawurlencode "$URL")
 
  # Our Result will contain ^M Line endings so we replace these with newline characters
 
 	(curl -s --user "$USER:$PASSWORD" "$URL" >/tmp/$link.result.txt)

  # We shouldnt, but sometimes we are seeing 408 Request Timeout so resubmit query if we do
  while  grep -Fxq "408 Request Timeout: Request Timeout" /tmp/$link.result.txt; 
  do
	#echo "408 Request Timeout: Request Timeout FOUND: " $link " Will retry "
	(curl -s --user "$USER:$PASSWORD" "$URL" >/tmp/$link.result.txt)
  done
  dos2unix -q -o /tmp/$link.result.txt
  
  COUNTER=1
  while [ $COUNTER -le $MAXVALUESPERKEY ]; do

	  OUTPUT=$link
	  OUTPUT+="^"
	  for meta in $METAKEYS
	  do
	    #cat /tmp/$link.result.txt |grep value |awk -v var=$meta -v count=$COUNTER '$0 ~ var {i++}i==count{print; exit}' >/tmp/$link.$meta.$COUNTER.output.txt
	    OUTPUT+=$(cat /tmp/$link.result.txt |grep value |awk  -v var=$meta -v count=$COUNTER '$0 ~ var {i++}i==count{print; exit}' |cut -d "=" -f 6 |sed 's/ type//' |xargs )
	    OUTPUT+="^" 
	  done
	  echo $OUTPUT |grep -v "$BLANKRESULT" >> $OUTPUTFEEDFILE$COUNTER$OUTPUTFEEDFILEEXT
	let COUNTER=COUNTER+1
  done
  rm -rf /tmp/$link.result.txt
  
done </tmp/linkfields.txt

