# Last edited on 2017-05-14 04:07:11 by stolfilocal

PROCESSING BITCOINCHARTS DAILY UPDATES

  # Extracting daily summary data of specific days from bitcoincharts 
  # 1d summaries to files called
  # ${DATE}-bitcoincharts-raw.txt
  
    extract_bitcoincharts_daily_updates.sh

  # Additional daily summary data is collected from bitcoincharts by hand into files
  # called 00-DATA/volumes/${DATE}-bitcoincharts-raw.txt, then processed with
  
    for mmdd in 03-{22,23,24,25,26,27,28,29,30} ; do 
      date="2014-${mmdd}"
      rawfile="${date}-bitcoincharts-raw.txt"
      kukfile="${date}-bitcoincharts.txt"
      echo "=== ${rawfile} --> ${kukfile} ===" 1>&2
      cat ${rawfile} \
        | cleanup_bitconcharts_daily_updates.sh ${date} \
        > ${kukfile}
    done
    
  # Beware that they do not contain all the same exchange/currency pairs.
  # Several pairs were added starting 2014-03-18.

  # Generating a column output for the daily volumes report:

    tabulate-volumes.sh 2014-03-{22,23,24,25,26,27,28,29,30}
        
CHECKING IF ALL PAIRS ARE INCLUDED

    cat relevant-ex-cr-pairs.dir | egrep -v -e '^ *[#]' > .relpairs
    for mmdd in 03-{22,23,24,25,26,27,28,29,30} ; do 
      date="2014-${mmdd}"
      echo "=== ${date} ==="
      gawk -v date="${date}" '/[|]/{ dt = $7; if (dt != date) { printf "** wrong date: %s\n", $0; }}' ${date}-bitcoincharts-raw.txt
      cat ${date}-bitcoincharts-raw.txt \
        | gawk '/[|]/{ ex=$1; cr=$3; printf "%s.%s\n", ex, cr; }' \
        | sort | uniq \
        | bool 1-2 .relpairs - \
        | map-field -v inField=1 -v outField=2 -v table=../../ex-cr-to-bitcoincharts.tbl -v defSubst="???" \
        | sed -e 's:[ ]*$::' -e 's:[ ]:|:' -e 's:[.]:|:' -e 's:$:|:' \
        | txtable-reformat \
        > .${date}-missing-data.txt
    done