:
#
#           get_datafiles = retrieve the logfiles and data files from archieves if required
#
#      L Isaksen   2001-06-13 fast ecfs option added
#      L Isaksen and Dick Dee 2004-06-13 Timeseries, varbc and reorganization
#
#############################################################################

set -a

#  Only plot the final obstat-plot with overlayed control or
#  in the case where no control is supplied

[ -d $EXEDIR ] || mkdir $EXEDIR
cd $EXEDIR

rc=0 

if [[ $VARBC_PARAMS = true ]] ; then    # retrieve main log files
  [ -d varbcparam ] || mkdir varbcparam
  cd varbcparam

  echo " "
  echo " Retrieve $EXPVER (stream $STREAM) main logfiles from ecfs to get varbc parameters"

  beg_varbc='begin varbc table 1'
  end_varbc='end varbc table 1'

  /bin/rm -f ./varbc${EXPVER}_satlist

  actualdate=$(echo $FIRSTCYCLE | cut -c1-8)

  if [[ $CLASS != "od" ]] ; then

    while [[ $actualdate -le $lastdate ]] ;
    do

      tarfile=main.${actualdate}.tar
      /bin/rm -f $tarfile $tarfile.Z
      echo "Getting $EXPVER $tarfile from ECFS.."
      if [[ $LMON_ERA = true ]] ; then
        ecp -o ec:/ERAS/era40/$EXPVER/log/an/$tarfile.Z .
      else
        ecp -o ec:/RDX/prepIFS/$EXPVER/log/an/$tarfile.Z .
      fi
      gunzip $tarfile.Z 

      tar -xf $tarfile
      /bin/rm -f $tarfile
      cycles=$(ls -d ??)
      for cycle in $cycles
      do
        if [[ -d ./$cycle/an/ifsvar ]] ; then
          #  Handles 3D-Var
          for file in $(ls ./$cycle/an/ifsvar/uptraj_0/ifsmin.?)
          do
            awk "/$beg_varbc/,/$end_varbc/" $file \
                 | sed -e '1 d' | sed -e "1,/$beg_varbc/ d" | sed -e '$ d' \
                 > ./varbc${EXPVER}${actualdate}${cycle}
            logfile=$file 
          done
        fi
        if [[ -d ./$cycle/an/4dvar ]] ; then
          #  Handles 4D-Var
          for file in $(ls ./$cycle/an/4dvar/uptraj_0/ifsmin.?)
          do
            awk "/$beg_varbc/,/$end_varbc/" $file \
                 | sed -e '1 d' | sed -e '$ d' \
                 > ./varbc${EXPVER}${actualdate}${cycle}
            logfile=$file 
          done
        fi
        if [[ -f $logfile ]] ; then
          #  update list of satellites
          awk '/JOT-sname:/,/%sensor:/' $logfile > ./satgrptable_1
          $DATADIR/varbc_satlist.pl < ./satgrptable_1 > ./satgrptable_2
          touch ./varbc${EXPVER}_satlist
          sort -u ./varbc${EXPVER}_satlist ./satgrptable_2 -o ./varbc${EXPVER}_satlist
        fi

        /bin/rm -r $cycle
      done

      actualdate=$(newdate -D $actualdate +1)
      
    done

   /bin/rm -f ./satgrptable_?

  else
    echo " "
    echo " varbc parameters not available in ops"
    exit 123
  fi

fi


if [[ $CONDITION_NUMBER = true ]] ; then    # retrieve main log files
  [ -d condition ] || mkdir condition
  cd condition

  echo ' 0.100' > missing_value

  rm -f condition_number?_${EXPVER}

  echo " "
  echo " Retrieve $EXPVER (stream $STREAM) main logfiles from ecfs to get condition number"

  if [[ "$STREAM" = 'DCDA' ]] ; then
   streampointer='dc'
  else
   streampointer=''
  fi

  actualdate=$(echo $FIRSTCYCLE | cut -c1-8)

  while [[ $actualdate -le $lastdate ]] ;
  do

    if [[ $CLASS != "od" ]] ; then
      tarfile=main.${actualdate}.tar
      /bin/rm -f $tarfile $tarfile.Z
      ecp -o ec:/RDX/prepIFS/$EXPVER/log/an/$tarfile.Z .
    else
      tarfile=main.${actualdate}.tar
      yyyymm=$(echo $actualdate | cut -c1-6)
      dd=$(echo $actualdate | cut -c7-8)
      
      /bin/rm -f $tarfile $tarfile.Z
      ecp -o ec:/emos/$EXPVER/o.main.${yyyymm}/${dd}.tar.Z $tarfile.Z
    fi 
    echo "Getting $EXPVER $tarfile from ECFS.."
    gunzip $tarfile.Z 

    tar -xf $tarfile
    /bin/rm -f $tarfile
    cycles=$(ls -d ??$streampointer)
    for cycle in $cycles
    do
      echo "cycle is $cycle"
      ls -l
      if [[ -d $cycle/an/4dvar ]] ; then
        grep 'congrad: ritz values are:'  $cycle/an/4dvar/uptraj_0/ifsmin.?  \
         | tail -1 | perl -ne 'chomp;print $1,"\n" if /(\d+\.\d+\s*$)/' > ritz0 
        if [[ -s ritz0 ]] ; then
          echo ${actualdate}${cycle} $(cat ritz0) >> condition_number0_$EXPVER
        else
          echo ${actualdate}${cycle} $(cat missing_value) >> condition_number0_$EXPVER
        fi
        echo condition_number0_$EXPVER
        cat condition_number0_$EXPVER

        grep 'congrad: ritz values are:'  $cycle/an/4dvar/uptraj_1/ifsmin.?  \
         | tail -1 | perl -ne 'chomp;print $1,"\n" if /(\d+\.\d+\s*$)/' > ritz1 
        if [[ -s ritz1 ]] ; then
          echo ${actualdate}${cycle} $(cat ritz1) >> condition_number1_$EXPVER
        else
          echo ${actualdate}${cycle} $(cat missing_value) >> condition_number1_$EXPVER
        fi
        echo condition_number1_$EXPVER
        cat condition_number1_$EXPVER

        ##echo ${actualdate}${cycle} $(grep -h 'congrad: converged eigenvalue' $cycle/an/4dvar/uptraj_0/ifsmin.? missing_value \
   ##| cut -c 31-66 | sort -n | tail -1 ) >> condition_number0_$EXPVER
        ##echo ${actualdate}${cycle} $(grep -h 'congrad: converged eigenvalue' $cycle/an/4dvar/uptraj_1/ifsmin.? missing_value \
   ##| cut -c 31-66 | sort -n | tail -1 ) >> condition_number1_$EXPVER
        /bin/rm -r $cycle
      fi
    done

    actualdate=$(newdate -D $actualdate +1)
      
  done

fi

cd $EXEDIR

if [[ $OBSTATPLOT = yes || $TIMESERIES = true ]] ; then    # 
  if [[ $USE_ECFSLOG = yes ]] ; then    # retrieve logfiles


    echo " "
    echo " Retrieve $EXPVER (stream $STREAM) logfiles from ecfs"
  
    actualdate=$(echo $FIRSTCYCLE | cut -c1-8)
  
    if [[ $CLASS != "od" ]] ; then

      while [[ $actualdate -le $lastdate ]] ;
      do

        if [[ $LMON_ERA = true ]] ; then
#          if [[ "$EXPVER" != 'BestERA' ]] ; then
#            ERA_EXPVER=$EXPVER
          if [[ $EXPVER = BestERAe4 ]] ; then
            if [[ $actualdate -lt 19570901 ]] ; then
              echo ERROR!! $actualdate before start of ERA-40
              exit 123
            elif [[ $actualdate -le 19571130 ]] ; then
              ERA_EXPVER=0272
            elif [[ $actualdate -le 19720430 ]] ; then
              ERA_EXPVER=0020
            elif [[ $actualdate -le 19721231 ]] ; then
              ERA_EXPVER=0334
            elif [[ $actualdate -le 19740430 ]] ; then
              ERA_EXPVER=0030
            elif [[ $actualdate -le 19740630 ]] ; then
              ERA_EXPVER=0335
            elif [[ $actualdate -le 19850331 ]] ; then
              ERA_EXPVER=0030
            elif [[ $actualdate -le 19860331 ]] ; then
              ERA_EXPVER=0050
            elif [[ $actualdate -le 19881031 ]] ; then
              ERA_EXPVER=0040
            elif [[ $actualdate -le 19881231 ]] ; then
              ERA_EXPVER=0338
            elif [[ $actualdate -le 19941231 ]] ; then
              ERA_EXPVER=0018
            elif [[ $actualdate -le 19950430 ]] ; then
              ERA_EXPVER=0274
            elif [[ $actualdate -le 20020831 ]] ; then
              ERA_EXPVER=0018
            else
              echo ERROR!! $actualdate after end of ERA-40
              exit 123
            fi
          elif [[ $EXPVER = BestERAei ]] ; then
            if [[ $actualdate -lt 19781001 ]] ; then
              echo ERROR!! $actualdate before start of ERA-Interim
              exit 123
            elif [[ $actualdate -le 19881231 ]] ; then
              ERA_EXPVER=1510
            elif [[ $actualdate -lt 19930831 ]] ; then
              ERA_EXPVER=1151
            elif [[ $actualdate -lt 20090131 ]] ; then
              ERA_EXPVER=1112
            elif [[ $actualdate -lt 20100131 ]] ; then
              ERA_EXPVER=1422
            else
              ERA_EXPVER=1473
            fi
          else
            ERA_EXPVER=$EXPVER
          fi
          FSBASE=$FSBASE_ERA/$ERA_EXPVER
        fi

        yyyy=$(echo $actualdate | cut -c1-4)
        mmdd=$(echo $actualdate | cut -c5-8)
  
        if [[ ! -f ${actualdate}.tar.Z ]] ; then
          if [[ "$STREAM" = 'DA' ]] ; then
            ecp -o ec:$FSBASE/ps/an/${actualdate}.tar.Z . ||  rc=$?
          else
            ecp -o ec:$FSBASE/ps/$STREAM/an/${actualdate}.tar.Z . ||  rc=$?
          fi
          if [[ "$rc" != 0 ]] ; then
            echo " Logfile for $CLASS experiment $EXPVER (stream $STREAM) on ${actualdate} not found!! "
            exit 123
          fi
        fi

        if [[ -f ${actualdate}.tar.Z ]] ; then
          /bin/rm -f ${actualdate}.tar
          gunzip ${actualdate}.tar.Z
        fi
      
        [ -f ${actualdate}.tar ] && tar -xf ${actualdate}.tar 
        /bin/rm -f ${actualdate}.tar *${actualdate}*obstat.ps
  
        actualdate=$(newdate -D $actualdate +1)
      done
      
      /bin/rm -f ${EXPVER}*.ps
    
    
      if [[ $EXPVER = 'BestERAe4' ]] ; then
        # Replace expver with "BestERAe4"
        $DATADIR/rename_file.pl 's/^..../BestERAe4/' *txt
      fi
      if [[ $EXPVER = 'BestERAei' ]] ; then
        # Replace expver with "BestERAei"
        $DATADIR/rename_file.pl 's/^..../BestERAei/' *txt
      fi
      $DATADIR/rename_file.pl 's/DCDAobstat\.txt//' *txt
      $DATADIR/rename_file.pl 's/DAobstat\.txt//' *txt
      $DATADIR/rename_file.pl 's/obstat\.txt//' *txt
      $DATADIR/rename_file.pl 's/^/stat/' $EXPVER*[00,06,12,18]
    
    else   # handle od type experiment
    

      newtypelogdate=20020409
      ecfsopslogdate=20020424
      ecfsopslogdate1=20030501
      newtypeesuitelogdate=20020225
      ecfsesuitelogdate=20020424
      ecfsesuitelogdate1=20030501

      if [[ $actualdate -lt $ecfsopslogdate ]] ; then
        ecd ec:/emos/$EXPVER
      fi

      while [[ $actualdate -le $lastdate ]] ;
      do
        yyyy=$(echo $actualdate | cut -c1-4)
        mmdd=$(echo $actualdate | cut -c5-8)
          mm=$(echo $actualdate | cut -c5-6)
          dd=$(echo $actualdate | cut -c7-8)
    
        if [[ ! -f ${actualdate}.tar.Z ]] ; then
          if [[ $EXPVER -eq 1 ]] ; then
            if [[ $actualdate -lt $newtypelogdate ]] ; then
              ecp -o ec:${yyyy}${mm}/${dd}.tar.Z ${actualdate}.tar.Z
            elif [[ $actualdate -lt $ecfsopslogdate ]] ; then
              ecp -o ec:o.lag.${yyyy}${mm}/${dd}.tar.Z ${actualdate}.tar.Z
            else
              if [[ $actualdate -lt $ecfsopslogdate1 ]] ; then
                ecp -o ec:/emos/pop/0001${actualdate}??${STREAM}obstat.txt .
              else
                if [[ "$STREAM" = 'DCDA' ]] ; then
                  ecp -o ec:/emos/DCDA/obstat/0001/${yyyy}${mm}/0001${actualdate}??${STREAM}obstat.txt . || rc=$?
                  if [[ "$rc" != 0 ]] ; then
                    echo "rc $rc "
                    sleep 12
                    echo " Obstat file ec:/emos/DCDA/obstat/0001/${yyyy}${mm}/0001${actualdate}??${STREAM}obstat.txt not found!! "
                    exit 123
                  else 
                    echo "rc OK  $rc "
                  fi
                else
                  ecp -o ec:/emos/obstat/0001/${yyyy}${mm}/0001${actualdate}??${STREAM}obstat.txt . || rc=$?
                  if [[ "$rc" != 0 ]] ; then
                    echo " Obstat file ec:/emos/obstat/0001/${yyyy}${mm}/0001${actualdate}??${STREAM}obstat.txt not found!! "
                    exit 123
                  fi
                fi
              fi
              for cycle in 00 06 12 18
              do
                if [[ -f 0001${actualdate}${cycle}${STREAM}obstat.txt ]] ; then
                  /bin/mv 0001${actualdate}${cycle}${STREAM}obstat.txt \
                       stat0001${actualdate}${cycle}
                fi
              done
            fi
          else   # e-suites
            if [[ $actualdate -lt $newtypeesuitelogdate ]] ; then
              ecp -o ec:${yyyy}${mm}/${dd}.tar.Z ${actualdate}.tar.Z
            elif [[ $actualdate -lt $ecfsesuitelogdate ]] ; then
              ecd ec:/emos/$EXPVER
              ecsfdir=$(els ec:e_*.lag.${yyyy}${mm} | cut -c 56-80)
              ecd $ecsfdir
              ecp -o ec:${dd}.tar.Z ${actualdate}.tar.Z
            else
              if [[ $actualdate -lt $ecfsesuitelogdate1 ]] ; then
                ecp -o ec:/emos/epop/${EXPVER}${actualdate}??${STREAM}obstat.txt .
              else
                if [[ "$STREAM" = 'DCDA' ]] ; then
                  ecp -o ec:/emos/DCDA/obstat/${EXPVER}/${yyyy}${mm}/${EXPVER}${actualdate}??${STREAM}obstat.txt .
                else
                  ecp -o ec:/emos/obstat/${EXPVER}/${yyyy}${mm}/${EXPVER}${actualdate}??${STREAM}obstat.txt .
                fi
              fi
              for cycle in 00 06 12 18
              do
                if [[ -f ${EXPVER}${actualdate}${cycle}${STREAM}obstat.txt ]] ; then
                  /bin/mv ${EXPVER}${actualdate}${cycle}${STREAM}obstat.txt \
                       stat${EXPVER}${actualdate}${cycle}
                fi
              done
            fi
          fi
        fi
        if [[ -f ${actualdate}.tar.Z ]] ; then
          /bin/rm -f ${actualdate}.tar
          gunzip ${actualdate}.tar.Z
        fi
    
        [ -f ${actualdate}.tar ] && tar -xf ${actualdate}.tar \
           00/plot/prms.1   00/plot/prms.2   00/plot/prms.3 \
           06/plot/prms.1   06/plot/prms.2   06/plot/prms.3 \
           12/plot/prms.1   12/plot/prms.2   12/plot/prms.3 \
           18/plot/prms.1   18/plot/prms.2   18/plot/prms.3 \
         ./00/plot/prms.1 ./00/plot/prms.2 ./00/plot/prms.3 \
         ./06/plot/prms.1 ./06/plot/prms.2 ./06/plot/prms.3 \
         ./12/plot/prms.1 ./12/plot/prms.2 ./12/plot/prms.3 \
         ./18/plot/prms.1 ./18/plot/prms.2 ./18/plot/prms.3
           /bin/rm -f ${actualdate}.tar ${actualdate}.tar.Z
   
        /bin/rm -rf fort.7
        for cycle in 00 06 12 18
        do
          [ -f $cycle/plot/prms.? ] && cat $cycle/plot/prms.? >> fort.7
          if [[ -s fort.7 ]]; then
            $DATADIR/${ARCH}_read_obstatlog.x > stat${EXPVER}${actualdate}${cycle}
          fi
          /bin/rm -rf fort.7 $cycle
        done
  
        actualdate=$(newdate -D $actualdate +1)
      done
  
    fi   

  else  # recalculate obstat statistics from feedback files

    if [[ $STACALC = 'no' || $STACALC = 'NO' ]] ; then
      echo " STACALC=no, so txt-files are expected to be available in $EXEDIR"
    else
      echo " You chose the slow option: to calculate obstat from feedback/ODB "
    fi

  fi

fi