* fixed bug in runtime extraction and added preliminary fast calculation of cached percentage

NaviXMon.py

@@ -75,7 +75,7 @@ class MonitorNaviX:
         # -> 3. test: multiple runs for more statistics (2 runs, 60 jobs, 10 files/job)
         navix_monitor_file_list = [r'plots/dev-null-test/04-tuned-SSDs-Raid0-bugfix/NaviX.mon.Copy.180818',
                                    r'plots/dev-null-test/04-tuned-SSDs-Raid0-bugfix/NaviX.mon.Copy.180819']
-        # navix_monitor_file_list = [r'plots/dev-null-test/04-tuned-SSDs-Raid0-bugfix/test.mon']
+        navix_monitor_file_list = [r'plots/dev-null-test/04-tuned-SSDs-Raid0-bugfix/test.mon']
         # Default copy:
         # navix_monitor_file_list = [r'data/copy/NaviX.mon.Copy.180818',
         #                            r'data/copy/NaviX.mon.Copy.180819']

navimon/NaviCommon.py

@@ -37,6 +37,7 @@ def ip_to_hostname(known_servers, ip):
     check if received IPs from pyxrootd belong to a known server side (based on input in knwon_host.yml file)
     """
     server_found = []
+    print(ip)
     for IP_range, server in iter(known_servers.items()):
         if is_ip_in_network_range(str((find_string_between(ip, "[", "]"))), IP_range):
             server_found.append(str(find_string_between(str(server.get("name")), "\"", "\"")))

navimon/NaviEvaluate.py

@@ -23,6 +23,11 @@ class EvaluateMonitoring:
         self.show_plots = False
         self.save_plots = True
 
+        try:
+            self.get_runtime()
+        except Exception as e:
+            print("Python-Error: " + str(e) + "\ncould not get runtime of jobs -> abort")
+            exit()
         try:
             self.get_data_rate()
         except Exception as e:
@@ -95,6 +100,13 @@ class EvaluateMonitoring:
                   + str(column))
             pass
 
+    def get_runtime(self):
+        try:
+            self.df['Runtime'] =np.where(self.df['Runtime'] > 0.0, self.df['Runtime'].astype(float), np.nan)
+        except Exception as e:
+            print("Python-Error: " + str(e) + "\nFailed to calculate waiting time!")
+            exit(-1)
+
     def get_data_rate(self):
         try:
             self.df['data_rate'] = self.df.loc[self.df['Runtime'] > 0.0]['NetworkInputMb'].divide(self.df['Runtime'])
@@ -111,6 +123,26 @@ class EvaluateMonitoring:
             exit(-1)
 
     def get_cached_percentage_on_worker_node(self):
+        # try:
+        #     pd.options.display.max_colwidth = 200
+        #     # print(self.df['files_found_on_server'])
+        #
+        #     self.df['cached_percentage_on_worker_node'] = self.df['files_found_on_server'].apply(lambda file_list: list(map(lambda key: dict(eval(str(file_list)))[key], dict(eval(str(file_list))).keys())))
+        #     print('Hallo', self.df['cached_percentage_on_worker_node'])
+        #
+        #     self.df['cached_percentage_on_worker_node'] = self.df['cached_percentage_on_worker_node'].apply(lambda server_list: [ip_to_hostname(self.known_servers, server) for server in server_list])
+        #     print('DU', self.df['cached_percentage_on_worker_node'])
+        #
+        #     # self.df['cached_percentage_on_worker_node'] = self.df['cached_percentage_on_worker_node'].apply(lambda server_list: list(filter(
+        #     #     lambda item: item != '[No server]', server_list)))
+        #     self.df['cached_percentage_on_worker_node'] = self.df[['LastRemoteHost', 'cached_percentage_on_worker_node']].apply(lambda combined_list: list(filter(
+        #         lambda file_list: combined_list[0] in file_list, eval(combined_list[1]))), axis=1)
+        #     print(self.df['cached_percentage_on_worker_node'])
+        #     exit(1)
+        # except Exception as e:
+        #     print("Python-Error: " + str(e) + "\nFailed to calculate cached percentage on worker node!")
+        #     exit(-1)
+
         self.df["cached_percentage_on_worker_node"] = 0.0
         for index, row in self.df.iterrows():
             cached_files = 0