Commit 1915e5eb authored by fwinkler's avatar fwinkler

Modified scalability scripts.

parent b6ed5770
...@@ -208,13 +208,19 @@ def send_data_to_db(influx_con, hostname, send_number, sample_rate): ...@@ -208,13 +208,19 @@ def send_data_to_db(influx_con, hostname, send_number, sample_rate):
#determine max duration of publish_data #determine max duration of publish_data
max_duration = get_max_duration(max_duration, write_end - write_start) max_duration = get_max_duration(max_duration, write_end - write_start)
if sent: if sent:
json_body = [] # new data block json_body = [] # new data block
elif len(json_body) < (5 * batch_size): elif len(json_body) < (5 * batch_size):
print ("Could not sent batch, buffering for next write!") print ("Could not sent batch, buffering for next write!")
else: else:
print ("Max sent request (buffer) exceeded! Drop metrics.") print ("Max sent request (buffer) exceeded! Drop metrics.")
json_body = [] json_body = []
#determine sleep time until next sample
if send_duration <= sample_rate:
sleep(sample_rate - send_duration)
else:
print (str((int(time.time()) - begin_time)/60) + "min': Write duration (" + str(send_duration) +") exceeded!")
node_iterator += 1 node_iterator += 1
if node_iterator == start_new_measurement: if node_iterator == start_new_measurement:
...@@ -224,14 +230,6 @@ def send_data_to_db(influx_con, hostname, send_number, sample_rate): ...@@ -224,14 +230,6 @@ def send_data_to_db(influx_con, hostname, send_number, sample_rate):
if send_iterator == send_number: if send_iterator == send_number:
return max_duration return max_duration
#determine sleep time until next sample
if send_duration <= sample_rate:
sleep(sample_rate - send_duration)
else:
print (str((int(time.time()) - begin_time)/60) + "min': Write duration (" + str(send_duration) +") exceeded!")
return max_duration return max_duration
......
#!/bin/bash #!/bin/bash
STRESS_HOST="172.24.136.90" export STRESS_HOST="172.24.146.84" #"172.24.136.90"
JOB_SCRIPT=${PWD}/job_stress_db.sh JOB_SCRIPT=${PWD}/job_stress_db.sh
NODE_NUM=10 #11 #14 #42 NODE_NUM=1 #10 #11 #14 #42
#determine measurement parameters #determine measurement parameters
SAMPLE_RATE=2 SAMPLE_RATE=3 #2
BATCH_SIZE=106 BATCH_SIZE=318 #106
VALUES_PER_MEASUREMENT=106 VALUES_PER_MEASUREMENT=106
SEND_NUMBER=$((60/SAMPLE_RATE * 120)) #120 simulation minutes SEND_NUMBER=$((60/SAMPLE_RATE * 2)) #120 simulation minutes
PROTOCOL="http" PROTOCOL="http" #"udp"
#fix settings #fix settings
NTASKS=24 NTASKS=24
...@@ -51,8 +51,9 @@ create_job_script() ...@@ -51,8 +51,9 @@ create_job_script()
run_stress_test() run_stress_test()
{ {
source /sw/taurus/tools/pika/pika-current.conf source /home/fwinkler/pika/pika-1.2.conf
COUNTS_1=$(get_measurement_counts) COUNTS_1=$(get_measurement_counts)
JOB_SUBMIT=`sbatch $JOB_SCRIPT` JOB_SUBMIT=`sbatch $JOB_SCRIPT`
JOB_ID=`echo $JOB_SUBMIT | cut -d " " -f 4` JOB_ID=`echo $JOB_SUBMIT | cut -d " " -f 4`
echo "$JOB_ID submitted" echo "$JOB_ID submitted"
...@@ -69,14 +70,24 @@ run_stress_test() ...@@ -69,14 +70,24 @@ run_stress_test()
cat $OUTPUT_PATH/stress_test_job.out cat $OUTPUT_PATH/stress_test_job.out
# get max send duration # get max send duration
# cat $OUTPUT_PATH/taurusi* | grep Duration | cut -d ":" -f 4 | sort -nr | head -n1 MAX_DURATION=`cat $OUTPUT_PATH/taurusi* | grep Duration | cut -d ":" -f 4 | sort -nr | head -n1`
echo "Max duration per simulation node: $MAX_DURATION"
COUNTS_2=$(get_measurement_counts) COUNTS_2=$(get_measurement_counts)
echo "Before: $COUNTS_1" echo "Before: $COUNTS_1"
echo "After: $COUNTS_2" echo "After: $COUNTS_2"
DIFF=$((COUNTS_2-COUNTS_1)) DIFF=$((COUNTS_2-COUNTS_1))
echo "Diff: $DIFF" EXPECTED=$((BATCH_SIZE*SEND_NUMBER*NODE_NUM*NTASKS))
echo "Expected values: $EXPECTED"
echo "Recorded values: $DIFF"
RESULT=$((EXPECTED-DIFF))
if [ "$RESULT" -eq 0 ] && [ "$MAX_DURATION" -le "$SAMPLE_RATE" ]; then
echo "Test completed successfully."
else
echo "Test failed."
fi
} }
clean_influx() clean_influx()
...@@ -90,8 +101,8 @@ clean_influx() ...@@ -90,8 +101,8 @@ clean_influx()
#curl -POST $INFLUX_CONF "q=CREATE DATABASE stress" #curl -POST $INFLUX_CONF "q=CREATE DATABASE stress"
#curl -POST $INFLUX_CONF "q=CREATE RETENTION POLICY longterm ON stress DURATION INF REPLICATION 1 SHARD DURATION 7d" #curl -POST $INFLUX_CONF "q=CREATE RETENTION POLICY longterm ON stress DURATION INF REPLICATION 1 SHARD DURATION 7d"
influxd restore -portable -db "prope" -newdb "stress" /tmp/566_1575244800 & influxd restore -portable -db "prope" -newdb "stress" /sw/taurus/tools/pika/test/stress_influx/shard_backup/566_1575244800 &
sleep 1 sleep 10
curl -POST $INFLUX_CONF "q=ALTER RETENTION POLICY frob ON stress DURATION inf SHARD DURATION 7d" curl -POST $INFLUX_CONF "q=ALTER RETENTION POLICY frob ON stress DURATION inf SHARD DURATION 7d"
wait wait
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment