Skip to content
Snippets Groups Projects
Commit 1f6bc7e7 authored by brusale's avatar brusale
Browse files

speed up event merging step

parent 4ec0c4c8
Branches
No related tags found
No related merge requests found
......@@ -59,6 +59,9 @@ def makeEvent(config_file, event=None):
def makeEventWrapper(args):
makeEvent(args[0], args[1])
def mergeCSVFilesWrapper(args):
mergeCSVFiles(args)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Generate samples')
......@@ -103,8 +106,10 @@ if __name__ == '__main__':
labels = [re.sub(pattern, '', csv_file) for csv_file in csv_files]
labels = list(set(labels))
with concurrent.futures.ProcessPoolExecutor(len(labels)) as executor:
for label in labels:
mergeCSVFiles(label, tmp_dir, csv_dir)
executor.submit(mergeCSVFilesWrapper, label)
stop = time.time()
mergeDelta = stop - start
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment