Added parallel execution

sciunit2/cli.py

@@ -62,6 +62,7 @@ def err2(msg1, msg2):
 
 
 def main():
+    print("debug_log")
     try:
         _main(sys.argv[1:])
     except CommandLineError:

sciunit2/command/exec_/__init__.py

@@ -6,6 +6,8 @@
 from sciunit2.util import path_injection_for
 import sciunit2.core
 import sciunit2.workspace
+import os
+import threading
 
 from getopt import getopt
 from pkg_resources import resource_filename
@@ -27,10 +29,20 @@ def run(self, args):
             raise CommandLineError
         emgr, repo = sciunit2.workspace.current()
         with emgr.exclusive():
-            rev = emgr.add(args)
             if optlist:
+                rev = emgr.add(args)
                 standin_fn = resource_filename(__name__, 'sciunit')
                 sciunit2.core.shell(env=path_injection_for(standin_fn))
+                results = self.do_commit('cde-package', rev, emgr, repo)
             else:
-                sciunit2.core.capture(args)
-            return self.do_commit('cde-package', rev, emgr, repo)
+                # make a new directory for the execution
+                thread_id = threading.current_thread().ident
+                directory_name = f"thread_{thread_id}"
+                os.makedirs(directory_name, exist_ok=True)
+
+                sciunit2.core.capture(args,cwd =directory_name)
+                results = self.do_commit_parallel(os.path.join(directory_name,'cde-package'), emgr, repo,args)
+
+                os.rmdir(directory_name)
+
+            return results

sciunit2/command/export.py

@@ -26,10 +26,13 @@ def python_paths(log_file, cde_root_dir):
     3. python version
     """
     with open(log_file) as f:
+        print(log_file)
+        print(cde_root_dir)
         line = f.readline()
         found = False
         pkg_dir_lines = []
         while line:
+            print(line)
             line = line.strip()
             # reading python executable path from the
             # first line after comments in the log file

sciunit2/command/mixin.py

@@ -3,6 +3,7 @@
 from sciunit2.util import quoted
 from sciunit2 import timestamp
 import sciunit2.workspace
+import threading
 
 from humanfriendly.terminal.spinners import Spinner
 
@@ -12,13 +13,25 @@
 # A spinning animation is displayed as feedback
 # to the end-user during the entire time.
 class CommitMixin(object):
+
+    lock = threading.Lock()
+
     def do_commit(self, pkgdir, rev, emgr, repo):
         with Spinner(label='Committing') as sp:
             # adds the execution to de-duplication engine
             sz = repo.checkin(rev, pkgdir, sp)
         # adds the execution to the database
         return (repo.location,) + emgr.commit(sz)
 
+    def do_commit_parallel(self, pkgdir, emgr, repo, args):
+        with self.lock :
+            rev = emgr.add(args)
+            with Spinner(label='Committing') as sp:
+                # adds the execution to de-duplication engine
+                sz = repo.checkin(rev, pkgdir, sp)
+            # adds the execution to the database
+            return (repo.location,) + emgr.commit(sz)
+
     def note(self, aList):
         return "\n[%s %s] %s\n Date: %s\n" % (
             sciunit2.workspace.project(aList[0]),

sciunit2/core.py

@@ -11,10 +11,10 @@
 
 
 # capture the execution of commands from cde.log
-def capture(args):
-    sciunit2.libexec.ptu(args).wait()
-    assert os.path.isdir('cde-package')
-    with open('cde-package/cde.log', 'r+') as f:
+def capture(args, cwd = None):
+    sciunit2.libexec.ptu(args,cwd=cwd).wait()
+    assert os.path.isdir(os.path.join(cwd,'cde-package'))
+    with open(os.path.join(cwd,'cde-package/cde.log'), 'r+') as f:
         f.prepend_cmd(args)
 
 

sciunit2/libexec/__init__.py

@@ -19,8 +19,8 @@ def __init__(self, name):
     def which(self):
         return pathfor(self.name)
 
-    def __call__(self, args, **kwargs):
-        return subprocess.Popen([self.which] + args, **kwargs)
+    def __call__(self, args,cwd=None, **kwargs):
+        return subprocess.Popen([self.which] + args,cwd=cwd, **kwargs)
 
 
 ptu = Process('ptu')