Index: utils/ds_score_plots/extract_data_from_log_file.py |
diff --git a/utils/ds_score_plots/extract_data_from_log_file.py b/utils/ds_score_plots/extract_data_from_log_file.py |
new file mode 100644 |
index 0000000000000000000000000000000000000000..708149cc29d0d3b7c364dbe88998a995372eba40 |
--- /dev/null |
+++ b/utils/ds_score_plots/extract_data_from_log_file.py |
@@ -0,0 +1,68 @@ |
+# -*- coding: utf-8 -*- |
+ |
+import pandas as pd |
+import os |
+import glob |
+from datetime import datetime |
+ |
+ |
+def read_log_file(filename, pattern): |
+ column = ['TIME', 'IP', 'SCORE'] |
+ df = pd.DataFrame(columns=column) |
+ with open(filename, 'r') as f: |
+ lines = f.readlines() |
+ |
+ for line in lines: |
+ line = line.rstrip() |
+ if line.find(pattern) != -1: |
+ info = line.split() |
+ time = info[2] + 'T' + info[3].replace(',', '.') |
+ time_sec = datetime.strptime(time, '%Y-%m-%dT%H:%M:%S.%f') |
+ # this will show the timestamp in ms |
+ timestamp_ms = time_sec.strftime("%s.%f")[:14] |
+ ip = info[-3] |
+ score = info[-1] |
+ title_of_list = {'TIME': timestamp_ms, 'IP': ip, 'SCORE': score} |
+ df = df.append(title_of_list, ignore_index=True) |
+ |
+ return df |
+ |
+ |
+# run with multiconf_sleep_time, Task 2.1 |
+def extract_data_from_log_cas(conf, |
+ output_dir=None, |
+ pattern='The new score of host'): |
+ log_file_dir = conf['env']['dir_scratch'] |
+ format_name_of_log_file = 'log.cas1.*' |
+ each_log_file = os.path.join(log_file_dir, 'runtime', |
+ format_name_of_log_file) |
+ log_files_to_enumerate = glob.glob(each_log_file) |
+ # use i,f to enumerate log files as the following format: |
+ # 1 log.cas1.1 |
+ # 2 log.cas1.2 |
+ # 3 log.cas1.3 |
+ # and write to a csv file |
+ for i, f in enumerate(log_files_to_enumerate): |
+ output_file_dir = os.path.join( |
+ output_dir, |
+ os.path.basename(f) + "_run_with_latency.csv") |
+ df = read_log_file(f, pattern) |
+ df.to_csv(output_file_dir, sep=',', index=False) |
+ |
+ |
+# run with multiconf_jitter_levels, Task 2.2 |
+def extract_data_from_log_cas_jitter(conf, |
+ output_dir=None, |
+ pattern='The new score of host'): |
+ log_file_dir = conf['env']['dir_scratch'] |
+ format_name_of_log_file = 'log.cas1.*' |
+ each_log_file = os.path.join(log_file_dir, 'runtime', |
+ format_name_of_log_file) |
+ log_files_to_enumerate = glob.glob(each_log_file) |
+ |
+ for i, f in enumerate(log_files_to_enumerate): |
+ output_file = os.path.join( |
+ output_dir, |
+ os.path.basename(f) + "_run_with_latency_and_jitter.csv") |
+ df = read_log_file(f, pattern) |
+ df.to_csv(output_file, sep=',', index=False, float_format='%10.3f') |