Skip to content
Snippets Groups Projects
Commit fa284060 authored by Naveen Kr. Sharma's avatar Naveen Kr. Sharma
Browse files

Adding tools for running experiments

parent 14c5ef37
No related branches found
No related tags found
No related merge requests found
import random
import string
import sys
charset = string.ascii_uppercase + string.ascii_lowercase + string.digits
for i in range(int(sys.argv[1])):
rkey = "".join(random.choice(charset) for j in range(64))
print rkey
import sys
start, end = -1.0, -1.0
duration = float(sys.argv[2])
warmup = duration/3.0
tLatency = []
sLatency = []
fLatency = []
tExtra = 0.0
sExtra = 0.0
fExtra = 0.0
xLatency = []
for line in open(sys.argv[1]):
if line.startswith('#') or line.strip() == "":
continue
line = line.strip().split()
if not line[0].isdigit() or len(line) < 4:
continue
if start == -1:
start = float(line[2]) + warmup
end = start + warmup
fts = float(line[2])
if fts < start:
continue
if fts > end:
break
latency = int(line[3])
status = int(line[4])
ttype = -1
try:
ttype = int(line[5])
extra = int(line[6])
except:
extra = 0
if status == 1 and ttype == 2:
xLatency.append(latency)
tLatency.append(latency)
tExtra += extra
if status == 1:
sLatency.append(latency)
sExtra += extra
else:
fLatency.append(latency)
fExtra += extra
if len(tLatency) == 0:
print "Zero completed transactions.."
sys.exit()
tLatency.sort()
sLatency.sort()
fLatency.sort()
print "Transactions(All/Success): ", len(tLatency), len(sLatency)
print "Abort Rate: ", (float)(len(tLatency)-len(sLatency))/len(tLatency)
print "Throughput (All/Success): ", len(tLatency)/(end-start), len(sLatency)/(end-start)
print "Average Latency (all): ", sum(tLatency)/float(len(tLatency))
print "Median Latency (all): ", tLatency[len(tLatency)/2]
print "Average Latency (success): ", sum(sLatency)/float(len(sLatency))
print "Median Latency (success): ", sLatency[len(sLatency)/2]
print "Extra (all): ", tExtra
print "Extra (success): ", sExtra
if len(xLatency) > 0:
print "X Transaction Latency: ", sum(xLatency)/float(len(xLatency))
if len(fLatency) > 0:
print "Average Latency (failure): ", sum(fLatency)/float(len(tLatency)-len(sLatency))
print "Extra (failure): ", fExtra
#!/bin/bash
trap '{
echo "\nKilling all clients.. Please wait..";
for host in ${clients[@]}
do
ssh $host "killall -9 $client";
ssh $host "killall -9 $client";
done
echo "\nKilling all replics.. Please wait..";
for host in ${servers[@]}
do
ssh $host "killall -9 server";
done
}' INT
# Paths to source code and logfiles.
srcdir="/homes/sys/naveenks/Research/Tapir"
logdir="/biggerraid/users/naveenks/tapir"
# Machines on which replicas are running.
replicas=("breakout" "pitfall" "qbert")
# Machines on which clients are running.
clients=("spyhunter")
client="benchClient" # Which client (benchClient, retwisClient, etc)
store="tapirstore" # Which store (strongstore, weakstore, tapirstore)
mode="txn-l" # Mode for storage system.
nshard=1 # number of shards
nclient=1 # number of clients to run (per machine)
nkeys=100000 # number of keys to use
rtime=10 # duration to run
tlen=2 # transaction length
wper=0 # writes percentage
err=0 # error
skew=0 # skew
zalpha=-1 # zipf alpha (-1 to disable zipf and enable uniform)
# Print out configuration being used.
echo "Configuration:"
echo "Shards: $nshard"
echo "Clients per host: $nclient"
echo "Threads per client: $nthread"
echo "Keys: $nkeys"
echo "Transaction Length: $tlen"
echo "Write Percentage: $wper"
echo "Error: $err"
echo "Skew: $skew"
echo "Zipf alpha: $zalpha"
echo "Skew: $skew"
echo "Client: $client"
echo "Store: $store"
echo "Mode: $mode"
# Generate keys to be used in the experiment.
echo "Generating random keys.."
python key_generator.py $nkeys > keys
# Start all replicas and timestamp servers
echo "Starting TimeStampServer replicas.."
$srcdir/store/tools/start_replica.sh tss $srcdir/store/tools/shard.tss.config \
"$srcdir/timeserver/timeserver" $logdir
for ((i=0; i<$nshard; i++))
do
echo "Starting shard$i replicas.."
$srcdir/store/tools/start_replica.sh shard$i $srcdir/store/tools/shard$i.config \
"$srcdir/store/$store/server -m $mode -f $srcdir/store/tools/keys -e $err -s $skew" $logdir
done
# Wait a bit for all replicas to start up
sleep 2
# Run the clients
echo "Running the client(s)"
count=0
for host in ${clients[@]}
do
ssh $host "$srcdir/store/tools/start_client.sh \"$srcdir/store/benchmark/$client \
-c $srcdir/store/tools/shard -N $nshard -f $srcdir/store/tools/keys \
-d $rtime -l $tlen -w $wper -k $nkeys -m $mode -e $err -s $skew -z $zalpha\" \
$count $nclient $logdir"
let count=$count+$nclient
done
# Wait for all clients to exit
echo "Waiting for client(s) to exit"
for host in ${clients[@]}
do
ssh $host "$srcdir/store/tools/wait_client.sh $client"
done
# Kill all replicas
echo "Cleaning up"
$srcdir/store/tools/stop_replica.sh $srcdir/store/tools/shard.tss.config > /dev/null 2>&1
for ((i=0; i<$nshard; i++))
do
$srcdir/store/tools/stop_replica.sh $srcdir/store/tools/shard$i.config > /dev/null 2>&1
done
# Process logs
echo "Processing logs"
cat $logdir/client.*.log | sort -g -k 3 > $logdir/client.log
rm -f $logdir/client.*.log
python $srcdir/store/tools/process_logs.py $logdir/client.log $rtime
f 1
replica breakout:51735
replica pitfall:51735
replica qbert:51735
f 1
replica breakout:51729
replica pitfall:51729
replica qbert:51729
f 1
replica breakout:51730
replica pitfall:51730
replica qbert:51730
f 1
replica breakout:51731
replica pitfall:51731
replica qbert:51731
f 1
replica breakout:51732
replica pitfall:51732
replica qbert:51732
f 1
replica breakout:51733
replica pitfall:51733
replica qbert:51733
#!/bin/bash
if [ "$#" -ne 4 ]; then
echo "Usage: $0 command begin_id ncopies" >&2
exit 1
fi
cmd=$1
begin=$2
copies=$3
logdir=$4
let end=$begin+$copies
for ((i=$begin; i<$end; i++))
do
command="$cmd > $logdir/client.$i.log 2>&1 &"
#echo $command
eval $command
done
#!/bin/bash
shard=$1 # which shard is this
config=$2 # path to config file
cmd=$3 # command to run
logdir=$4 # log directory
if [ "$#" -ne 4 ]; then
echo "Usage: $0 shard configpath command logdir" >&2
exit 1
fi
n=$(head -n1 $config | awk '{print $2}')
let n=2*$n+1
for ((i=0; i<$n; i++))
do
let line=$i+2
server=$(cat $config | sed -n ${line}p | awk -F'[ :]' '{print $2}')
command="ssh $server \"$cmd -c $config -i $i > $logdir/$shard.replica$i.log 2>&1 &\""
#echo $command
eval $command
done
#!/bin/bash
config=$1 # path to config file
if [ "$#" -ne 1 ]; then
echo "Usage: $0 configpath" >&2
exit 1
fi
n=$(head -n1 $config | awk '{print $2}')
let n=2*$n+1
for ((i=0; i<$n; i++))
do
let line=$i+2
server=$(cat $config | sed -n ${line}p | awk -F'[ :]' '{print $2}')
command="ssh $server \"pkill -INT server\""
#echo $command
eval $command
done
#/bin/bash
procname=$1
check=1
while [ $check -gt 0 ]
do
check=`pgrep -u $USER -x $procname | wc -l`
sleep 1
done
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment