* added some modifications to save input files to classAd of condor -> grid...

* added some modifications to save input files to classAd of condor -> grid contorl is sometimes to fast with clean up

config/HTCondor_bindings.cnf

@@ -8,7 +8,7 @@ working_directory = Iwd
 requested_files = Input_Files
 
 # classad containing requested files (current: "Input_Files") as tmp backup in case of gridcontrol
-requested_files = InputFiles_tmp
+requested_files_tmp = InputFiles_tmp
 
 # classad to set a rank for the job -> ranking is set from Rank + HPDA_RANK
 #                                      (current: "HPDA_RANK", set by JobRouter/NaviX)
@@ -80,4 +80,4 @@ LastRemoteHost = LastRemoteHost
 # classad created by HTCondor containing the ip of the executing worker node
 #                   (used for monitoring)
 #                   (current: "StartdPrincipal")
-LastRemoteIP = StartdPrincipal
\ No newline at end of file
+LastRemoteIP = StartdPrincipal

hooks/lightHook_finalize.cpp

@@ -116,6 +116,7 @@ int main() {
         return -1;
     }
     requested_files = requested_files + "END_OF_REQUESTED_FILES\n";
+    InputFiles_tmp = InputFiles_tmp + "END_OF_REQUESTED_FILES\n";
     // open connection
     client = open_connection(portNum);
     if (client == -1){

navix/monitor.py

@@ -97,15 +97,13 @@ class Monitor:
             elif self.settings.requested_files_tmp in line[:len(self.settings.requested_files_tmp)] \
                     or next_line_input_files:
                 try:
-                    if self.settings.requested_files + " = " in line[:len(self.settings.requested_files + " = ")]:
-                        line = line.replace(self.settings.requested_files + " = \"", "")
-                    if '\"END_OF_REQUESTED_FILES' in line[-24:]:
+                    if self.settings.requested_files_tmp + " = " in line[:len(self.settings.requested_files_tmp + " = ")]:
+                        line = line.replace(self.settings.requested_files_tmp + " = \"", "")
+                    if 'END_OF_REQUESTED_FILES' in line[-len('END_OF_REQUESTED_FILES'):]:
                         next_line_input_files = False
-                        self.Input_Files.append(line[:-23])
                     else:
                         next_line_input_files = True
-                        self.Input_Files.append(line)
-
+                        self.Input_Files.extend(line.split(", "))
                 except Exception as e:
                     self.queue_logging.put("ERROR: " + str(self.thread_num)
                                            + ": Failed to get list of input files (string with server name)!\n"
@@ -136,11 +134,10 @@ class Monitor:
                                            + ": Failed to get requirement string!\n"
                                            + "Python-ERROR: " + str(e))
         if self.Input_Files:
-            self.get_file_size()
             if 'Input_Files = "' == str(self.Input_Files[0][len('Input_Files = "'):]):
                 self.Input_Files[0] = self.Input_Files[0][len('Input_Files = "'):]
             pass
-
+            self.get_file_size()
         self.queue_monitor.put(str(self.timestamp) + ": \n"
                                + "LastRemoteHost: " + self.remote_host + "\n"
                                + "LastRemoteIP: " + self.remote_ip + "\n"
@@ -182,6 +179,7 @@ class Monitor:
                                + str(self.Input_Files) + ": \n"
                                + "###########################################")
         for index, file_selected in enumerate(self.Input_Files):
+            file_selected = file_selected.replace("\"","").replace("\'","")
             try:
                 self.queue_logging.put("DEBUG: " + str(self.thread_num) + ": "
                                        + "Requesting statistic data for file: " + str(file_selected)

navix/reply_generator.py

@@ -50,7 +50,7 @@ class ComputeAndGenerateReply:
 
         # fill list with requested files for further processing -> e.g. searching the files via pyxrootd
         try:
-            files_to_search_array = create_file_list_to_search(self.queue_logging, self.thread_num, self.data)
+            files_to_search_array, input_tmp = create_file_list_to_search(self.queue_logging, self.thread_num, self.data)
         except Exception as err:
             raise ValueError('Failed to cast required files into a list!'
                              + 'ERROR: ' + str(err))
@@ -112,7 +112,7 @@ class ComputeAndGenerateReply:
             raise ValueError('Failed in creating response!')
 
         try:
-            reply += "InputFiles_tmp = "  + str(files_to_search_array)
+            reply += "InputFiles_tmp = \""  + str(input_tmp) + "\"\n"
         except Exception as e:
             self.queue_logging.put("ERROR: " + str(self.thread_num)
                                    + ": Failed while adding InputFiles_tmp! -> Abort!"

navix/replygenerator/generate_file_list.py

@@ -65,6 +65,15 @@ def get_files_to_search_from_data(queue_logging, thread_num, data):
         queue_logging.put("WARNI: " + str(thread_num) + ": Failed to remove double \" from: "
                           + str(data)
                           + "\nPython-ERROR: " + str(e))
+    # save requested files with prefix
+    try:
+        input_tmp = str(data).replace("\n", ", ")
+    except Exception as e:
+         queue_logging.put("ERROR: " + str(thread_num)
+                              + ": Failed to save input files to local variable, data: "
+                              + str(data)
+                              + "\nPython-ERROR: " + str(e))
+         raise
     while True:
         # check if files are separated by ","
         try:
@@ -123,4 +132,4 @@ def get_files_to_search_from_data(queue_logging, thread_num, data):
                           + str(("/" + ((data.split('root://', 1)[1]).split("//", 1)[1])))
                           + " to " + str(files_to_search_array)
                           + "\nPython-ERROR: " + str(e))
-    return files_to_search_array
+    return files_to_search_array, input_tmp