Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(887)

Unified Diff: utils/ds_score_plots/extract_data_from_log_file.py

Issue 339190043: Plot from all snitches
Patch Set: Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: utils/ds_score_plots/extract_data_from_log_file.py
diff --git a/utils/ds_score_plots/extract_data_from_log_file.py b/utils/ds_score_plots/extract_data_from_log_file.py
new file mode 100644
index 0000000000000000000000000000000000000000..708149cc29d0d3b7c364dbe88998a995372eba40
--- /dev/null
+++ b/utils/ds_score_plots/extract_data_from_log_file.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+import pandas as pd
+import os
+import glob
+from datetime import datetime
+
+
+def read_log_file(filename, pattern):
+ column = ['TIME', 'IP', 'SCORE']
+ df = pd.DataFrame(columns=column)
+ with open(filename, 'r') as f:
+ lines = f.readlines()
+
+ for line in lines:
+ line = line.rstrip()
+ if line.find(pattern) != -1:
+ info = line.split()
+ time = info[2] + 'T' + info[3].replace(',', '.')
+ time_sec = datetime.strptime(time, '%Y-%m-%dT%H:%M:%S.%f')
+ # this will show the timestamp in ms
+ timestamp_ms = time_sec.strftime("%s.%f")[:14]
+ ip = info[-3]
+ score = info[-1]
+ title_of_list = {'TIME': timestamp_ms, 'IP': ip, 'SCORE': score}
+ df = df.append(title_of_list, ignore_index=True)
+
+ return df
+
+
+# run with multiconf_sleep_time, Task 2.1
+def extract_data_from_log_cas(conf,
+ output_dir=None,
+ pattern='The new score of host'):
+ log_file_dir = conf['env']['dir_scratch']
+ format_name_of_log_file = 'log.cas1.*'
+ each_log_file = os.path.join(log_file_dir, 'runtime',
+ format_name_of_log_file)
+ log_files_to_enumerate = glob.glob(each_log_file)
+ # use i,f to enumerate log files as the following format:
+ # 1 log.cas1.1
+ # 2 log.cas1.2
+ # 3 log.cas1.3
+ # and write to a csv file
+ for i, f in enumerate(log_files_to_enumerate):
+ output_file_dir = os.path.join(
+ output_dir,
+ os.path.basename(f) + "_run_with_latency.csv")
+ df = read_log_file(f, pattern)
+ df.to_csv(output_file_dir, sep=',', index=False)
+
+
+# run with multiconf_jitter_levels, Task 2.2
+def extract_data_from_log_cas_jitter(conf,
+ output_dir=None,
+ pattern='The new score of host'):
+ log_file_dir = conf['env']['dir_scratch']
+ format_name_of_log_file = 'log.cas1.*'
+ each_log_file = os.path.join(log_file_dir, 'runtime',
+ format_name_of_log_file)
+ log_files_to_enumerate = glob.glob(each_log_file)
+
+ for i, f in enumerate(log_files_to_enumerate):
+ output_file = os.path.join(
+ output_dir,
+ os.path.basename(f) + "_run_with_latency_and_jitter.csv")
+ df = read_log_file(f, pattern)
+ df.to_csv(output_file, sep=',', index=False, float_format='%10.3f')
« no previous file with comments | « utils/ds_score_plots/copy_ping_latency_data_to_unique_folder.py ('k') | utils/ds_score_plots/fetch_score_and_write_to_csv.py » ('j') | no next file with comments »

Powered by Google App Engine
RSS Feeds Recent Issues | This issue
This is Rietveld f62528b