* NaviXMon is now skipping read in of data if temporary data from last run is...

* NaviXMon is now skipping read in of data if temporary data from last run is valid and code has not changed

NaviXMon.py

@@ -5,7 +5,7 @@ import os
 import glob
 
 from navimon.NaviEvaluate import EvaluateMonitoring
-from navimon.NaviCommon import test_and_transform_ip
+from navimon.NaviCommon import test_and_transform_ip, check_hashes
 from navimon.NaviPlot import *
 
 
@@ -38,10 +38,6 @@ def __init__():
     export_as = '.pdf'
     # export_as = '.png'
 
-    known_servers = read_server_list()
-    if known_servers == -1:
-        exit()
-
     # ==============================================================================================================
 
     # Plot various tests
@@ -99,37 +95,51 @@ def __init__():
     # --------
     # navix_monitor_file_list = [r'NaviX.mon']
 
-    get_data_from_file(monitor_dict, navix_monitor_file_list)
+    # skip read in if saved dataframe is up to date
+    check_file_list = ['NaviXMon.py', 'navimon/NaviCommon.py', 'navimon/NaviEvaluate.py']
+    check_file_list += navix_monitor_file_list
 
-    monitor_df = dict_to_pandas(monitor_dict)
-    evaluate_monitor = EvaluateMonitoring(monitor_df, df_file, known_servers)
+    if check_hashes(check_file_list):
+        print('Skipping read in, using saved dataframes')
+        df_job = pd.read_csv(".df_job.csv", sep=';', encoding='utf-8')
+        df_file = pd.read_csv(".df_file.csv", sep=';', encoding='utf-8')
+
+    else:
+        known_servers = read_server_list()
+        if known_servers == -1:
+            exit()
+        get_data_from_file(monitor_dict, navix_monitor_file_list)
+        df_job = dict_to_pandas(monitor_dict)
+        evaluate_monitor = EvaluateMonitoring(df_job, df_file, known_servers)
+        df_job.to_csv(".df_job.csv", sep=';', encoding='utf-8')
+        df_file.to_csv(".df_file.csv", sep=';', encoding='utf-8')
 
-    print("\nRuntime for getting and preparing data: " + str(np.around(time() - start, decimals=2)) + 's')
+        print("\nRuntime for getting and preparing data: " + str(np.around(time() - start, decimals=2)) + 's')
 
-    evaluate_monitor.print_some_mean_err()
+        evaluate_monitor.print_some_mean_err()
 
     start = time()
 
-    plot_hist_cache_rate(monitor_df, export_as)
-    plot_hist_cpu_efficiency(monitor_df, export_as)
-    plot_hist_data_rate(monitor_df, export_as)
-    plot_hist_runtime(monitor_df, export_as)
-    plot_hist_file_size(monitor_df, export_as)
+    plot_hist_cache_rate(df_job, export_as)
+    plot_hist_cpu_efficiency(df_job, export_as)
+    plot_hist_data_rate(df_job, export_as)
+    plot_hist_runtime(df_job, export_as)
+    plot_hist_file_size(df_job, export_as)
     plot_hist_duplicity(df_file, export_as)
-    plot_scatter_data_rate_vs_runtime(monitor_df, export_as)
-    plot_scatter_file_size_sum_vs_data_rate(monitor_df, export_as)
-    plot_scatter_data_rate_vs_cpu_efficiency(monitor_df, export_as)
-    plot_scatter_average_waiting_time_vs_runtime(monitor_df, export_as)
+    plot_scatter_data_rate_vs_runtime(df_job, export_as)
+    plot_scatter_file_size_sum_vs_data_rate(df_job, export_as)
+    plot_scatter_data_rate_vs_cpu_efficiency(df_job, export_as)
+    plot_scatter_average_waiting_time_vs_runtime(df_job, export_as)
     plot_scatter_file_size_vs_duplicity(df_file, export_as)
     plot_lines_file_duplicity_vs_time(df_file, export_as)
-    plot_errorbar_file_percentage_vs_throughput(monitor_df, export_as)
-    plot_errorbar_file_percentage_vs_cpu_efficiency(monitor_df, export_as)
-    plot_function_processing_time_vs_cached_percentage(monitor_df, export_as)
+    plot_errorbar_file_percentage_vs_throughput(df_job, export_as)
+    plot_errorbar_file_percentage_vs_cpu_efficiency(df_job, export_as)
+    plot_function_processing_time_vs_cached_percentage(df_job, export_as)
 
     # old version of plots:
-    # plot_kde_data_rate_vs_cpu_efficiency(monitor_df)
+    # plot_kde_data_rate_vs_cpu_efficiency(df_job)
     # plot_kde_file_size_vs_duplicity(df_file)
-    # plot_errorbar_file_percentage_vs_processing_time(monitor_df)
+    # plot_errorbar_file_percentage_vs_processing_time(df_job)
 
     print("Runtime for plotting and reviewing graphs: " + str(np.around(time() - start, decimals=2)) + 's')
     print('##################################################################')

navimon/NaviCommon.py

 from ipaddress import ip_address, ip_network
+import json
+import hashlib
 
 
 c_red = '\033[91m'
@@ -97,3 +99,27 @@ def translate_node_to_cache(server, known_servers):
             if is_ip_in_network_range(server.split('/')[0], ip):
                 tmp_list.append(key)
     return tmp_list
+
+
+def check_hashes(check_file_list):
+    result = True
+    new_test_dict = dict()
+    block_size = 65536
+    hash_library = hashlib.md5()
+    for file in check_file_list:
+        with open(file, 'rb') as afile:
+            buf = afile.read(block_size)
+            while len(buf) > 0:
+                hash_library.update(buf)
+                buf = afile.read(block_size)
+        new_test_dict[file] = hash_library.hexdigest()
+    try:
+        test_dict = json.load(open(".hashes.tmp"))
+        for key, value in iter(new_test_dict.items()):
+            if test_dict[key] != value:
+                result = False
+                break
+    except Exception as e:
+        result = False
+    json.dump(new_test_dict, open(".hashes.tmp", 'w'))
+    return result
\ No newline at end of file

navimon/NaviEvaluate.py

@@ -71,7 +71,6 @@ class EvaluateMonitoring:
             print(c_red + "Python-Error: " + str(e) + "\nERROR while getting file duplicity -> abort" + c_end)
             exit()
 
-        self.df.to_csv("debug.csv", sep=';', encoding='utf-8')
         self.df_cached_node = self.df[self.df.cached_percentage_on_worker_node != 0]
         self.df_not_cached = self.df[self.df.cached_percentage_on_worker_node == 0]