#!/bin/sh # # process_minutes.sh # # Perform Q transform based analysis on 10 minutes of science mode data. # # usage: # # process_minutes.sh startTime [triggers] [event] [properties] [report] # # The requested startTime may be in GPS seconds or a format understood # by the LIGOTools tconvert utility. # # The remaining optional arguments specify whether or not trigger production, # a qscan of the loudest event, plots of trigger properties, or an html report # is produced. By default nothing is produced. # Shourov K. Chatterji # shourov@ligo.caltech.edu # $Id: process_minutes.sh,v 1.18 2007/08/25 11:03:15 shourov Exp $ ################################################################################ # Setup environment # ################################################################################ # change to directory containing this script cd `dirname $0` # path to Q transform installation Q=../.. # setup environment . ${Q}/bin/qsetup.sh ################################################################################ # Command line arguments # ################################################################################ # usage information if [ $# -eq 0 ]; then echo "usage: `basename $0` startTime [triggers] [event] [properties] [report]" 1>&2 exit 1 fi # analysis start time echo "$1" | grep -qv [^0-9] >/dev/null if [ $? -eq 0 ]; then startTime="$1" else startTime=`tconvert "$1"` fi # requested products if [ $# -gt 1 ]; then shift 1 requested="$*" else requested="" fi ################################################################################ # Timing information # ################################################################################ # analysis time string truncated to nearest minute startYear=`tconvert -f "%Y" ${startTime}` startMonth=`tconvert -f "%m" ${startTime}` startDay=`tconvert -f "%d" ${startTime}` startHour=`tconvert -f "%H" ${startTime}` startMinute=`tconvert -f "%M" ${startTime}` startTimeString=`tconvert -f "%Y-%m-%d %H:%M:00" ${startTime}` # analysis start time truncated to nearest minute startTime=`tconvert "${startTimeString}"` # analysis duration in minutes analysisDuration="10" # analysis stop time stopTime=`expr ${startTime} + 60 \* ${analysisDuration}` # minimum segment duration in seconds minimumSegmentDuration=`grep blockDuration parameters.txt | \ awk ' { print int($2) } '` # append minimum segment duration to analysis stop time stopTime=`expr ${stopTime} + ${minimumSegmentDuration}` # time of previous minutes previousMinutes=`tconvert "${startYear}-${startMonth}-${startDay} \ ${startHour}:${startMinute}:00 UTC" - \ ${analysisDuration}minutes` previousMinutesYear=`tconvert -f "%Y" ${previousMinutes}` previousMinutesMonth=`tconvert -f "%m" ${previousMinutes}` previousMinutesDay=`tconvert -f "%d" ${previousMinutes}` previousMinutesHour=`tconvert -f "%H" ${previousMinutes}` previousMinutesMinute=`tconvert -f "%M" ${previousMinutes}` # time of next minutes nextMinutes=`tconvert "${startYear}-${startMonth}-${startDay} \ ${startHour}:${startMinute}:00 UTC" + \ ${analysisDuration}minutes` nextMinutesYear=`tconvert -f "%Y" ${nextMinutes}` nextMinutesMonth=`tconvert -f "%m" ${nextMinutes}` nextMinutesDay=`tconvert -f "%d" ${nextMinutes}` nextMinutesHour=`tconvert -f "%H" ${nextMinutes}` nextMinutesMinute=`tconvert -f "%M" ${nextMinutes}` # time of previous hour previousHour=`tconvert "${startYear}-${startMonth}-${startDay} \ ${startHour}:00:00 UTC" - 1hour` previousHourYear=`tconvert -f "%Y" ${previousHour}` previousHourMonth=`tconvert -f "%m" ${previousHour}` previousHourDay=`tconvert -f "%d" ${previousHour}` previousHourHour=`tconvert -f "%H" ${previousHour}` # time of next hour nextHour=`tconvert "${startYear}-${startMonth}-${startDay} \ ${startHour}:00:00 UTC" + 1hour` nextHourYear=`tconvert -f "%Y" ${nextHour}` nextHourMonth=`tconvert -f "%m" ${nextHour}` nextHourDay=`tconvert -f "%d" ${nextHour}` nextHourHour=`tconvert -f "%H" ${nextHour}` # time of previous day previousDay=`tconvert "${startYear}-${startMonth}-${startDay} \ 00:00:00 UTC" - 1day` previousDayYear=`tconvert -f "%Y" ${previousDay}` previousDayMonth=`tconvert -f "%m" ${previousDay}` previousDayDay=`tconvert -f "%d" ${previousDay}` # time of next day nextDay=`tconvert "${startYear}-${startMonth}-${startDay} \ 00:00:00 UTC" + 1day` nextDayYear=`tconvert -f "%Y" ${nextDay}` nextDayMonth=`tconvert -f "%m" ${nextDay}` nextDayDay=`tconvert -f "%d" ${nextDay}` ################################################################################ # Detector information # ################################################################################ # gravitational wave channel name # *** THIS NEEDS TO BE UPDATED TO HANDLE MULTIPLE DETECTORS *** channelName=`grep channelName parameters.txt | \ awk ' { print $2 } ' | sed -e "s|[{}']||g"` # interferometer identifier # *** THIS NEEDS TO BE UPDATED TO HANDLE MULTIPLE DETECTORS *** interferometer=`echo ${channelName} | sed -e 's|:.*$||'` ################################################################################ # Directory structure # ################################################################################ # day directory name dayDirectory="${startYear}/${startMonth}/${startDay}" # hour directory name hourDirectory="${dayDirectory}/${startHour}" # minutes directory name minutesDirectory="${hourDirectory}/${startMinute}" # create output directory mkdir -p ${minutesDirectory} # initialize log file date -u +"%Y-%m-%d %H:%M:%S %Z" >>${minutesDirectory}/log.txt ################################################################################ # Trigger production # ################################################################################ # begin if triggers requested if [ -n "`echo ${requested} | grep -e triggers`" ]; then # remove existing results if [ -d ${minutesDirectory}/log ]; then echo "removing existing analysis results..." >>${minutesDirectory}/log.txt rm -rf ${minutesDirectory}/log rm -rf ${minutesDirectory}/results fi # copy parameter file echo "copying parameter file..." >>${minutesDirectory}/log.txt cp parameters.txt ${minutesDirectory}/parameters.txt # determine science mode segments echo "getting science mode segments..." >>${minutesDirectory}/log.txt getsegments.sh ${interferometer} ${startTime} ${stopTime} \ >${minutesDirectory}/segments.txt # read segment list echo "reading segment list..." >>${minutesDirectory}/log.txt segments=`sed -e 's| |-|' ${minutesDirectory}/segments.txt` # begin loop over segments for segment in ${segments}; do # extract segment start time segmentStartTime=`echo ${segment} | sed -e 's|-.*$||'` # extract segment stop time segmentStopTime=`echo ${segment} | sed -e 's|^.*-||'` # run qpipeline on segment echo "analyzing ${segment}..." >>${minutesDirectory}/log.txt ${Q}/bin/qpipeline.sh ${segmentStartTime} ${segmentStopTime} \ ${minutesDirectory}/parameters.txt \ @online \ ${minutesDirectory} # end loop over segments done # collect triggers # *** THIS NEEDS TO BE UPDATED TO HANDLE MULTIPLE DETECTORS *** echo "collecting triggers..." >>${minutesDirectory}/log.txt cat ${minutesDirectory}/results/${channelName}_*.txt \ 2>/dev/null | grep -v '%' >${minutesDirectory}/triggers.txt # apply vetoes # *** THIS NEEDS TO BE UPDATED TO HANDLE MULTIPLE DETECTORS *** # down-select triggers echo "down-selecting triggers..." >>${minutesDirectory}/log.txt ${Q}/postprocess/bin/select ${minutesDirectory}/triggers.txt \ ${minutesDirectory}/triggers.txt.tmp mv ${minutesDirectory}/triggers.txt.tmp ${minutesDirectory}/triggers.txt # cluster triggers echo "clustering triggers..." >>${minutesDirectory}/log.txt ${Q}/postprocess/bin/select 1.0 \ ${minutesDirectory}/triggers.txt \ ${minutesDirectory}/clusters.txt # sort triggers by time sort -g ${minutesDirectory}/triggers.txt >${minutesDirectory}/triggers.txt.tmp sort -g ${minutesDirectory}/clusters.txt >${minutesDirectory}/clusters.txt.tmp mv ${minutesDirectory}/triggers.txt.tmp ${minutesDirectory}/triggers.txt mv ${minutesDirectory}/clusters.txt.tmp ${minutesDirectory}/clusters.txt # collect livetimes echo "collecting livetimes..." >>${minutesDirectory}/log.txt cat ${minutesDirectory}/results/livetime_*.txt 2>/dev/null | \ awk ' { print $2, $3 } ' | sort -g \ >${minutesDirectory}/livetime.txt # merge livetimes echo "merging livetimes..." >>${minutesDirectory}/log.txt segexpr ${minutesDirectory}/livetime.txt ${minutesDirectory}/livetime.txt.tmp mv ${minutesDirectory}/livetime.txt.tmp ${minutesDirectory}/livetime.txt # end if triggers requested fi ################################################################################ # Loudest event properties # ################################################################################ # begin if loudest event qscan, trigger properties, or html report requested if [ -n "`echo ${requested} | grep -e event -e properties -e report`" ]; then # find loudest event echo "finding loudest event..." >>${minutesDirectory}/log.txt loudestEventProperties=`sort -gk 5,5 ${minutesDirectory}/clusters.txt \ 2>/dev/null | tail -n 1` if [ -n "${loudestEventProperties}" ]; then loudestEventTime=`echo ${loudestEventProperties} | awk ' { print $1 } '` loudestEventFrequency=`echo ${loudestEventProperties} | awk ' { print $2 } '` loudestEventDuration=`echo ${loudestEventProperties} | awk ' { print $3 } '` loudestEventBandwidth=`echo ${loudestEventProperties} | awk ' { print $4 } '` loudestEventSNR=`echo ${loudestEventProperties} | awk ' { printf "%9.3e", sqrt(2 * $5) } '` fi # end if loudest event qscan, trigger properties, or html report requested fi ################################################################################ # Loudest event QScan # ################################################################################ # begin if loudest event qscan requested if [ -n "`echo ${requested} | grep -e event`" ]; then # remove existing loudest event qscan existingLoudestEvent=`find ${minutesDirectory}/loudestevent -printf "%l" 2>/dev/null` if [ -n "${existingLoudestEvent}" ]; then echo "removing existing loudest event scan..." >>${minutesDirectory}/log.txt rm -rf ${minutesDirectory}/${existingLoudestEvent} fi # perform loudest event qscan # *** THIS NEEDS TO BE UPDATED TO HANDLE MULTIPLE DETECTORS *** rm -f ${minutesDirectory}/loudestevent rm -f ${minutesDirectory}/loudestevent_thumbnail.png if [ -n "${loudestEventProperties}" ]; then echo "scanning loudest event..." >>${minutesDirectory}/log.txt ln -s ${loudestEventTime} ${minutesDirectory}/loudestevent ${Q}/bin/qscan.sh ${loudestEventTime} \ configuration_minutes.txt \ @online \ ${minutesDirectory} convert -resize 350x350 \ ${minutesDirectory}/loudestevent/${loudestEventTime}_${channelName}_1.00_spectrogram_whitened.png \ ${minutesDirectory}/loudestevent_thumbnail.png fi # end if loudest event qscan requested fi ################################################################################ # Trigger properties # ################################################################################ # begin if trigger properties requested if [ -n "`echo ${requested} | grep -e properties`" ]; then # produce trigger properties # *** THIS NEEDS TO BE UPDATED TO HANDLE MULTIPLE DETECTORS *** echo "plotting trigger properties..." >>${minutesDirectory}/log.txt ${Q}/bin/qproperties.sh ${startTime} ${stopTime} \ ${minutesDirectory}/parameters.txt \ ${minutesDirectory}/triggers.txt \ ${minutesDirectory}/clusters.txt \ ${minutesDirectory}/livetime.txt \ ${channelName} \ ${minutesDirectory}/properties # end if trigger properties requested fi ################################################################################ # HTML report # ################################################################################ # begin if html report requested if [ -n "`echo ${requested} | grep -e report`" ]; then # current time stamp currentTime=`date -u "+%Y-%m-%d %H:%M:%S %Z"` # create minutes html report echo "creating html report..." >>${minutesDirectory}/log.txt rm -f ${minutesDirectory}/style.css ln -s ../../../../../style.css ${minutesDirectory}/style.css sed -e "s|\[START_YEAR\]|${startYear}|g" \ -e "s|\[START_MONTH\]|${startMonth}|g" \ -e "s|\[START_DAY\]|${startDay}|g" \ -e "s|\[START_HOUR\]|${startHour}|g" \ -e "s|\[START_MINUTE\]|${startMinute}|g" \ -e "s|\[PREVIOUSMINUTES_YEAR\]|${previousMinutesYear}|g" \ -e "s|\[PREVIOUSMINUTES_MONTH\]|${previousMinutesMonth}|g" \ -e "s|\[PREVIOUSMINUTES_DAY\]|${previousMinutesDay}|g" \ -e "s|\[PREVIOUSMINUTES_HOUR\]|${previousMinutesHour}|g" \ -e "s|\[PREVIOUSMINUTES_MINUTE\]|${previousMinutesMinute}|g" \ -e "s|\[NEXTMINUTES_YEAR\]|${nextMinutesYear}|g" \ -e "s|\[NEXTMINUTES_MONTH\]|${nextMinutesMonth}|g" \ -e "s|\[NEXTMINUTES_DAY\]|${nextMinutesDay}|g" \ -e "s|\[NEXTMINUTES_HOUR\]|${nextMinutesHour}|g" \ -e "s|\[NEXTMINUTES_MINUTE\]|${nextMinutesMinute}|g" \ -e "s|\[CURRENT_TIME\]|${currentTime}|g" \ -e "s|\[ANALYSIS_DURATION\]|${analysisDuration}|g" \ -e "s|\[INTERFEROMETER\]|${interferometer}|g" \ -e "s|\[LOUDEST_EVENT_TIME\]|${loudestEventTime}|g" \ -e "s|\[LOUDEST_EVENT_FREQUENCY\]|${loudestEventFrequency}|g" \ -e "s|\[LOUDEST_EVENT_DURATION\]|${loudestEventDuration}|g" \ -e "s|\[LOUDEST_EVENT_BANDWIDTH\]|${loudestEventBandwidth}|g" \ -e "s|\[LOUDEST_EVENT_SNR\]|${loudestEventSNR}|g" \ -e "s|\[CHANNEL_NAME\]|${channelName}|g" \ ../html/index_minutes.html >${minutesDirectory}/index.html # create placeholder hour html report if [ ! -f ${hourDirectory}/index.html ]; then echo "creating placeholder hour html report..." >>${minutesDirectory}/log.txt rm -f ${hourDirectory}/style.css ln -s ../../../../style.css ${hourDirectory}/style.css sed -e "s|\[START_YEAR\]|${startYear}|g" \ -e "s|\[START_MONTH\]|${startMonth}|g" \ -e "s|\[START_DAY\]|${startDay}|g" \ -e "s|\[START_HOUR\]|${startHour}|g" \ -e "s|\[PREVIOUSHOUR_YEAR\]|${previousHourYear}|g" \ -e "s|\[PREVIOUSHOUR_MONTH\]|${previousHourMonth}|g" \ -e "s|\[PREVIOUSHOUR_DAY\]|${previousHourDay}|g" \ -e "s|\[PREVIOUSHOUR_HOUR\]|${previousHourHour}|g" \ -e "s|\[NEXTHOUR_YEAR\]|${nextHourYear}|g" \ -e "s|\[NEXTHOUR_MONTH\]|${nextHourMonth}|g" \ -e "s|\[NEXTHOUR_DAY\]|${nextHourDay}|g" \ -e "s|\[NEXTHOUR_HOUR\]|${nextHourHour}|g" \ -e "s|\[CURRENT_TIME\]|${currentTime}|g" \ -e "s|\[ANALYSIS_DURATION\]|${analysisDuration}|g" \ -e "s|\[INTERFEROMETER\]|${interferometer}|g" \ -e "s|\[LOUDEST_EVENT_TIME\]||g" \ -e "s|\[LOUDEST_EVENT_FREQUENCY\]||g" \ -e "s|\[LOUDEST_EVENT_DURATION\]||g" \ -e "s|\[LOUDEST_EVENT_BANDWIDTH\]||g" \ -e "s|\[LOUDEST_EVENT_SNR\]||g" \ -e "s|\[CHANNEL_NAME\]|${channelName}|g" \ ../html/index_hour.html >${hourDirectory}/index.html fi # create placeholder day html report if [ ! -f ${dayDirectory}/index.html ]; then echo "creating placeholder day html report..." >>${minutesDirectory}/log.txt rm -f ${dayDirectory}/style.css ln -s ../../../style.css ${dayDirectory}/style.css sed -e "s|\[START_YEAR\]|${startYear}|g" \ -e "s|\[START_MONTH\]|${startMonth}|g" \ -e "s|\[START_DAY\]|${startDay}|g" \ -e "s|\[PREVIOUSDAY_YEAR\]|${previousDayYear}|g" \ -e "s|\[PREVIOUSDAY_MONTH\]|${previousDayMonth}|g" \ -e "s|\[PREVIOUSDAY_DAY\]|${previousDayDay}|g" \ -e "s|\[NEXTDAY_YEAR\]|${nextDayYear}|g" \ -e "s|\[NEXTDAY_MONTH\]|${nextDayMonth}|g" \ -e "s|\[NEXTDAY_DAY\]|${nextDayDay}|g" \ -e "s|\[CURRENT_TIME\]|${currentTime}|g" \ -e "s|\[ANALYSIS_DURATION\]|${analysisDuration}|g" \ -e "s|\[INTERFEROMETER\]|${interferometer}|g" \ -e "s|\[LOUDEST_EVENT_TIME\]||g" \ -e "s|\[LOUDEST_EVENT_FREQUENCY\]||g" \ -e "s|\[LOUDEST_EVENT_DURATION\]||g" \ -e "s|\[LOUDEST_EVENT_BANDWIDTH\]||g" \ -e "s|\[LOUDEST_EVENT_SNR\]||g" \ -e "s|\[CHANNEL_NAME\]|${channelName}|g" \ ../html/index_day.html >${dayDirectory}/index.html fi # update link to most recent results lastStopTime=`awk ' END { printf "%d", $2 } ' \ latest_minutes/livetime.txt 2>/dev/null` thisStopTime=`awk ' END { printf "%d", $2 } ' \ ${minutesDirectory}/livetime.txt 2>/dev/null` if [ ${thisStopTime:-0} -ge ${lastStopTime:-0} ]; then echo "linking html report..." >>${minutesDirectory}/log.txt rm -f latest_minutes ln -s ${minutesDirectory} latest_minutes fi # end if html report requested fi ################################################################################ # End # ################################################################################ # return to calling function echo "done." >>${minutesDirectory}/log.txt exit 0