rawurlencode() {
  local string="${1}"
  local strlen=${#string}
  local encoded=""
  local pos c o

  for (( pos=0 ; pos<strlen ; pos++ )); do
     c=${string:$pos:1}
     case "$c" in
        [-_.~a-zA-Z0-9] ) o="${c}" ;;
        * )               printf -v o '%%%02x' "'$c"
     esac
     encoded+="${o}"
  done
  echo "${encoded}"    # You can either set a return variable (FASTER) 
  REPLY="${encoded}"   #+or echo the result (EASIER)... or both... :p
}
#Copy the existing Feed to a backup location
cp /var/www/html/useragentfeed.csv /var/www/html/useragentfeed.csv.bak
# We keep a list all UserAgents that we have processed in /tmp/processed.txt

touch /tmp/processed.txt
# First Get a list of UserAgents from our Broker. We are only concerned with those that are http traffic
#curl -s --user 'admin:netwitness' 'http://192.168.123.249:50103/sdk?msg=values&fieldName=client&where=service=80&size=2000' |grep field |cut -f 2 -d ">" |cut -d "<" -f1 >|>tmp/useragent.txt
curl -s --user 'admin:netwitness' 'http://192.168.123.249:50103/sdk?msg=values&fieldName=client&size=20000' |grep field |cut -f 2 -d ">" |cut -d "<" -f1 |grep -v rsyslogd | grep -v pts |grep -v ignored |grep -v \(\) >/tmp/useragent.txt
#For each UserAgent send it through our device detector Parser

while read p; do
  cmd=$(grep -ci "$p" /tmp/processed.txt)
  escape_p=$(echo "$p" |sed 's/\[/\\[/')
  cmd2=$(grep -ci "$escape_p" /tmp/processed.txt)
  if [ $cmd == "0" ] && [ $cmd2 == "0" ]; then
    #echo  "UserAgent "$p"  not previously seen so process it"
    echo  "$p" >>/tmp/processed.txt
    php -f /var/www/html/device-detector-master/script.php "$p" >>/var/www/html/useragentfeed.csv
  #else
  #  echo "UserAgent already in processed"
  fi
done </tmp/useragent.txt
