# Last edited on 2017-05-14 04:10:42 by stolfilocal
  
FETCHED HUOBI'S TRADE LOGS FROM GITHUB

  # On 2014-07-18 I downdloaded the Huobi transaction logs from {github},
  # with "Download ZIP" from
  
    https://github.com/huobi/btc-trade-result-history

SAVING TRANSACTION LOGS

  # Extracted the transaction log of Bitcoinwisdom chart.
  # The translaction log was obtained doing CTRL-A CTRL-C on the chart,
  # pasting into a file, and running it through
  # {cleanup_bitcoinwisdom_trans_log.gawk}, e.g.
  
    cleanup_bitcoinwisdom_trans_log.gawk -v dt=2014-01-21 | txtable-reformat
    
  # Beware of change of date in the middle of a segment, fix as needed.
  
  # This leaves the entries in reverse chronological order (most recent first).
  # Use {tac} to reverse them.
  
  # Beware that the most recent entry may be incomplete.  If one waited a bit longer
  # before the copy-paste, its values might have been different.
  # For example, here is the result of two successive copy-pastes:
  
    # Copy1:
    2014-05-07 02:38:36 | 2691.00 |  4.0855
    2014-05-07 02:38:35 | 2690.99 |  1.3497
    2014-05-07 02:38:32 | 2691.00 |  0.0339
    # ...

    # Copy2:
    2014-05-07 02:38:52 | 2692.01 |  0.0043
    2014-05-07 02:38:47 | 2692.01 |  0.0627
    2014-05-07 02:38:38 | 2691.00 | 19.5422
    2014-05-07 02:38:35 | 2690.99 |  1.3497
    # ...
    
  # Note that the first line "02:38:36 | 2691.00 | 4.0855" of Copy1 got replaced by a merged line
  # "02:38:38 | 2691.00 | 19.5422" in Copy2, with same price, larger volume, and slightly later 
  # time. 
  #
  # However, there are many examples of succesive lines with same price and even same timestamp
  # that are NOT merged, e.g.

    2014-05-07 02:30:06 | 2689 | 2.5423
    2014-05-07 02:30:06 | 2689 | 0.0227
    
  # Perhaps merging only happens when successive entries with same price
  # have the same buyer and seller, or when the passive order is the same,
  # or when the time gaps are much less than one second, or whatever. 
 
  # Until early 2014, Bitcoinwisdom worked only with the user's local time,
  # so in the earliest captured logs I had to fix the times (and sometimes dates)
  # by hand to convert from local time to UTC.  That was delicate since 
  # the local time switched from daylight savings to normal in mid-february.
  # Hope I got all right.

SAVING LONGER TRANSACTION LOGS

  # To save a longer segment of the log, one must repeat
  # the copy-paste multiple times, pasting all the segments in 
  # sequence into the same file, and run {cleanup_bitcoinwisdom_trans_log.gawk}
  # on each segment. The segments must overlap by at least 2 entries.
  # Beware of change of date in the middle of a segment, fix as needed.
  
  # Then run {remove_dup_trans.gawk} on the whole file.  It removes extra headers,
  # marks duplicate entries with "### " prefix, and reverses the order of entries
  # in each segment so that they are in chrono order (most recent last).
  
  # Finally run the result through {egrep -v -e '^[\#][\#][\#]' | txtable-reformat}
  # to obtain the clean file.
      
  # !!! TO DO: Check again whether all raw files have been converted properly without errors.
  
  # !!! TO DO: FIX the bugs in the following files:

  # Checking for unconverted logs:
  
    check_cleanup_status.sh
      
  # Converting them:
  
    check_cleanup_status.sh
    for arg in \
      `cat .raws-linux | sed -e 's/-\([0-9][0-9][0-9][0-9]\)-/:\1:/g'` \
    ; do
      fld=( `echo "${arg}" | tr ':' ' '` )
      dt="${fld[0]}"
      tm="${fld[1]}"
      ex="${fld[2]}"
      name="${dt}-${tm}-${ex}"
      fraw="raw/linux/${name}-raw.txt"
      fout="clean/${name}.txt"
      echo "=== ${fraw} --> ${fout} ==="
      remove_dup_trans.gawk  ${fraw} > .all.txt || rm -fv .all.txt
      if [[ -s .all.txt ]]; then
        egrep -v -e '^[\#][\#][\#]' .all.txt \
          | txtable-reformat \
          > ${fout}
      else
        rm -fv ${fout}
      fi
    done
    check_cleanup_status.sh

SAVING LONGER LOGS FROM APPLE

  # When transaction logs are copy-pasted from the Apple Safari browser, the 
  # format is a bit different.  In that case we need to do
  
  #   2014-05-19:0720:HUBI \
  #   2014-05-21:0515:HUBI \
  #   2014-05-21:0728:HUBI \

    check_cleanup_status.sh
    for arg in \
      `cat .raws-apple | sed -e 's/-\([0-9][0-9][0-9][0-9]\)-/:\1:/g'` \
    ; do
      fld=( `echo "${arg}" | tr ':' ' '` )
      dt="${fld[0]}"
      tm="${fld[1]}"
      ex="${fld[2]}"
      name="${dt}-${tm}-${ex}"
      fraw="raw/apple/${name}-raw-apple.txt"
      fout="clean/${name}.txt"
      echo "=== ${fraw} --> ${fout} ==="
      cleanup_bitcoinwisdom_trans_logs_apple.sh "${dt}" "${tm}" "${ex}"
    done
    check_cleanup_status.sh

GETTING DATE RANGES

  # Getting their date ranges:
  
    check_cleanup_status.sh
    rm -f .date-ranges
    for name in `cat .cleans` ; do
      cfile="clean/${name}.txt"
      printf "${cfile}\n" 1>&2
      if [[ -s ${cfile} ]]; then 
        cat ${cfile} | egrep -e '^2014' | gawk '//{ print $1, $2 }' | sort > .tmp
        printf "%s %s %s\n" "${name}" "`head -n 1 .tmp`" "`tail -n 1 .tmp`" >> .date-ranges
      else
        printf "%s ** not found\n" "${cfile}" 1>&2
      fi
    done
  
RENAMING FILES

  # Long time ago, the files were named as follows

    "clean/{DATE}-{HHMM}-{EX}.txt"
    "clean/{DATE}-{HHMM}-{EX}-{TAG}.txt"
    "raw/linux/{{DATE}-{HHMM}-{EX}-raw.txt"
    "raw/apple/{{DATE}-{HHMM}-{EX}-raw-apple.txt"

  # However the file sometimes spanned midnight,
  # and the {HHMM} part was quite inconsistent.
  # On 2014-06-27 all files were renamedmore systematically as 
  
      "{DIR}/{INIDATE}-{INIHM}--{FINDATE}-{FINHM}-{EX}-{CR}-{TAG}.txt"
      
  # Where {DIR} is "clean", "raw/linux", or "raw/apple" respectively,
  # and {TAG} is "raw", "raw-apple" for the raw files, and
  # the original "{TAG}" for the clean file, or "all" if there was no 
  # {TAG} on the latter.
  
    make_rename_cmds.gawk .date-ranges > .rename_cmds
    do_rename_files.sh
  
PLOTTING INTERVALS SPANNED

  # Plotting the time intervals spanned by the various files:

    for ex in HUBI MGOX BSTP ; do 
      for cr in CNY USD EUR ; do 
        for tu in all ; do 
          files=( `ls *--*-${ex}-${cr}-${tu}.txt || echo NONE` )
          if [[ "/${files[0]}" != "/NONE" ]]; then
            echo "=== ${files[0]} ==="
            ../show_date_intervals.sh clean "${ex}" "${cr}" "${tu}" \
              > plots/date-intervals-${ex}-${cr}-${tu}.png
          fi
        done
      done
    done

PLOTTING TRANSACTION LOGS

  # Plotting transactions:
  
    plot_transactions.sh \
        "Bitstamp, BTC-e and Huobi transactions 2014-01-17 around 23:50 UTC" \
        0.0 3 775.0 825.0 50.0 \
        clean/2014-01-17-2320--2014-01-17-2358-BSTP-USD-all.txt "Bitstamp"  1.000 \
        clean/2014-01-17-2350--2014-01-18-0002-BTCE-USD-all.txt "BTC-e"     1.000 \
        clean/2014-01-17-2354--2014-01-18-0005-HUBI-CNY-all.txt "Huobi"     6.000 \
      > plots/2014-01-17-2350--2014-01-18-0005-BSTP-BTCE-HUBI-trans.png
  
    plot_transactions.sh \
        "Huobi transactions 2014-01-18 around 02:10 UTC" \
        0.0 2 812.0 818.0 60.0 \
        clean/2014-01-18-0218--2014-01-18-0228-HUBI-CNY-all.txt "Huobi"     6.000 \
      > plots/2014-01-18-0218--2014-01-18-0228-HUBI-trans.png
      
    plot_transactions.sh \
        "Transactions 2014-01-21 around 16:32 UTC" \
        0.0 6 815.0 895.0 50.0 \
        clean/2014-01-21-1613--2014-01-21-1633-BSTP-USD-all.txt "Bitstamp"  1.000 \
        clean/2014-01-21-1533--2014-01-21-1630-MGOX-USD-all.txt "MtGOX"     1.150 \
        clean/2014-01-21-1618--2014-01-21-1634-BTCE-USD-all.txt "BTC-e"     1.000 \
        clean/2014-01-21-1633--2014-01-21-1635-HUBI-CNY-all.txt "Huobi"     6.000 \
        clean/2014-01-21-1623--2014-01-21-1635-OKCO-CNY-all.txt "OKCoin"    6.000 \
      > plots/2014-01-21-1533--2014-01-21-1635-BSTP-MGOX-BTCE-HUBI-OKCO-trans.png
      
    plot_transactions.sh \
        "Transactions 2014-01-21 around 16:32 UTC" \
        0.0 6 815.0 845.0 50.0 \
        clean/2014-01-21-1613--2014-01-21-1633-BSTP-USD-all.txt "Bitstamp"  1.000 \
        clean/2014-01-21-1618--2014-01-21-1634-BTCE-USD-all.txt "BTC-e"     1.000 \
        clean/2014-01-21-1633--2014-01-21-1635-HUBI-CNY-all.txt "Huobi"     6.000 \
        clean/2014-01-21-1623--2014-01-21-1635-OKCO-CNY-all.txt "OKCoin"    6.000 \
      > plots/2014-01-21-1613--2014-01-21-1635-BSTP-BTCE-HUBI-OKCO-trans.png
    
    plot_transactions.sh \
        "Transactions MtGOX and Bitstamp 2014-01-22 around 01:00 UTC" \
        0.0 6 815.0 835.0 50.0 \
        clean/2014-01-22-0030--2014-01-22-0146-BSTP-USD-all.txt "Bitstamp"  1.000 \
        clean/2014-01-22-0032--2014-01-22-0144-MGOX-USD-all.txt "MtGOX"     1.170 \
      > plots/2014-01-22-0030--2014-01-22-0146-BSTP-MGOX-trans.png

    plot_transactions.sh \
        "Transactions 2014-01-22 around 19:00 UTC" \
        0.0 6 805.0 865.0 50.0 \
        clean/2014-01-22-1839--2014-01-22-1917-BSTP-USD-all.txt "Bitstamp"  1.000 \
        clean/2014-01-22-1840--2014-01-22-1915-HUBI-CNY-all.txt "Huobi"     6.000 \
        clean/2014-01-22-1836--2014-01-22-1919-MGOX-USD-all.txt "MtGOX"     1.110 \
        clean/2014-01-22-1846--2014-01-22-1919-BTCE-USD-all.txt "BTC-e"     1.000 \
        clean/2014-01-22-1846--2014-01-22-1919-BTCE-USD-all.txt "BTC-e"     1.000 \
        clean/2014-01-22-1848--2014-01-22-1919-OKCO-CNY-all.txt "OKCoin"    6.000 \
      > plots/2014-01-22-1836--2014-01-22-1919-BSTP-BTCE-MGOX-HUBI-OKCO-trans.png

    plot_transactions.sh \
        "Transactions 2014-03-04 around 01:30 UTC" \
        0.0 6 605.0 705.0 50.0 \
        clean/2014-03-04-0052--2014-03-04-0159-BSTP-USD-all.txt "Bitstamp"  1.000 \
        clean/2014-03-04-0146--2014-03-04-0201-HUBI-CNY-all.txt "Huobi"     6.100 \
      > plots/2014-03-04-0052--2014-03-04-0201-BSTP-HUBI-trans.png
    
    plot_transactions.sh \
        "Transactions 2014-03-09 around 05:14 UTC" \
        0.0 6 3690.0 3705.0 10.0 \
        clean/2014-03-09-0508--2014-03-09-0517-OKCO-CNY-all.txt "OKCoin"  1.000 \
        clean/2014-03-09-0511--2014-03-09-0522-HUBI-CNY-all.txt "Huobi"   1.000 \
      > plots/2014-03-09-0508--2014-03-09-0522-OKCO-HUBI-trans.png
    
    plot_transactions.sh \
        "Transactions 2014-05-26 around 09:38 UTC" \
        0.0 6 3530.0 3560.0 50.0 \
        clean/2014-05-26-0128--2014-05-26-0130-HUBI-CNY-all.txt "Huobi"   1.000 \
      > plots/2014-05-26-0128--2014-05-26-0130-HUBI-trans.png

    plot_transactions.sh \
        "Transactions 2014-05-26 around 02:45 UTC" \
        0.0 6 3570.0 3630.0 50.0 \
        clean/2014-05-26-0247--2014-05-26-0302-HUBI-CNY-all.txt "Huobi"   1.000 \
      > plots/2014-05-26-0247--2014-05-26-0302-HUBI-trans.png


THE MTGOX PING-PONG ROBOT

  # In January 2014 there was a robot in MtGOX creating a steady stream
  # of 0.01 BTC transactions, for unknown reason.
  # 
  # Took a 10-minute segment of the transaction log.
  # Separated the apparent ping-pong volume (0.01 BTC transactions) from 
  # transactions at other volumes.  Files are
  # 
  #   2014-01-17-1800-MGOX-pingpong.txt
  #   2014-01-17-1800-MGOX-legit.txt
  
  # Plotting them:
  
    plot_transactions.sh \
        "MtGOX transactions 2014-01-17 around 20:00 UTC" \
        0.0 1 875.0 915.0 4.0 \
        clean/2014-01-17-1957--2014-01-17-2005-MGOX-USD-pingpong.txt "(0.01 BTC)"  1.000 \
        clean/2014-01-17-1957--2014-01-17-2005-MGOX-USD-legit.txt    "(other)"     1.000 \
      > plots/2014-01-17-1957--2014-01-17-2005-MGOX-trans.png
    
THE HUOBI PING-PONG ROBOT

  # On May 19, 2014 there was a robot at Huobi creating a steady stream
  # of 0.01 BTC transactions, similar to the MtGOX ping-pong robot.
  # 
  # Took a 20-minute segment of the transaction log:
  
    cat clean/2014-05-18-2328--2014-05-19-0120-HUBI-CNY-all.txt \
      | gawk '/^2014/ { tm=$2; if ((tm >= "23:20:00") && (tm < "23:40:00")) { print; } } ' \
      > .temp.txt
    wc -l .temp.txt
  
  # Separated the apparent ping-pong volume (0.01 BTC transactions) from 
  # transactions at other volumes
  
    cat .temp.txt \
      | gawk '//{ vb = $6; if (vb+0 == 0.01) { print; } }' \
      > .temp-pingpong.txt
  
    cat .temp.txt \
      | gawk '//{ vb = $6; if (vb+0 != 0.01) { print; } }' \
      > .temp-legit.txt
  
  # Plotting them:
  
    plot_transactions.sh \
        "Huobi transactions 2014-05-18 around 23:30 UTC" \
        0.0 1 2760 2770 4.0 \
        .temp-pingpong.txt "(0.01 BTC)"  1.000 \
        .temp-legit.txt    "(other)"     1.000 \
      > plots/2014-05-18-2328--2014-05-19-0120-HUBI-trans.png
    
CHECKING TRANSACTION FREQUENCY

  # Counting transactions per unit time:
  # !!! FIX NAMES
  
      for ex in BSTP BTCE HUBI MGOX OKCO ; do 
        echo "=== ${ex} ==="
        compute_trans_per_minute.gawk clean/2014-01-21-1632-${ex}.txt
      done
  
      for ex in BSTP BTCE HUBI MGOX OKCO ; do 
        echo "=== ${ex} ==="
        compute_trans_per_minute.gawk clean/2014-01-22-1900-${ex}.txt
      done
  
      for ex in BSTP HUBI ; do 
        echo "=== ${ex} ==="
        compute_trans_per_minute.gawk clean/2014-03-04-0100-${ex}.txt
      done
  
      for ex in OKCO HUBI ; do 
        echo "=== ${ex} ==="
        compute_trans_per_minute.gawk clean/2014-03-09-0514-${ex}.txt
      done