making sure that parallel jobs are cleaned up after running

main/run_map_parallel.py

@@ -26,7 +26,7 @@ met_data_location = '/gpfs/ts0/projects/Research_Project-148395/s2p3_rv2.0/test/
 met_data_temporary_location = base_directory+'met/spatial_data/' # The location that met data for each year will be un tar.gziped into
 start_year = 2006
 end_year = 2006 # same as start year resuls in a 1 year run
-depth_min = 10 # NOTE that these numbers MUST be the same as those used in the scripts used to produce the meterology and nutrient files, otherwse data will not be taken for teh correct lats/lons and/or the script will fail 
+depth_min = 10 # NOTE that these numbers MUST be the same as those used in the scripts used to produce the meterology and nutrient files, otherwse data will not be taken for teh correct lats/lons and/or the script will fail
 depth_max = 100
 write_error_output = False
 
@@ -130,7 +130,7 @@ def run_model(domain_file_name,lats_lons,year,start_year,unique_job_id,met_data_
     if lon_domain_tmp < 0.0:
         lon_domain_tmp = 360.0+lon_domain_tmp
     # forcing_lat_lon = closest(lats_lons, float(lat_domain[i]),float(lon_domain[i]))
-    print i
+    # print i
     # print str(forcing_lat_lon[0])
     # print str(forcing_lat_lon[1])
     # print str(float(lat_domain[i]))
@@ -241,8 +241,8 @@ unique_job_id = str(uuid.uuid4())
 num_lines = sum(1 for line in open(base_directory+'domain/'+domain_file_name)) - 1
 # num_lines = 10
 
-
-subprocess.call('tar -C '+met_data_temporary_location+' -zxvf '+met_data_location+'met_data_'+str(start_year)+'.tar.gz', shell=True)
+print 'initial unzipping of met data to extrcat lat and lon data'
+subprocess.call('tar -C '+met_data_temporary_location+' -zxf '+met_data_location+'met_data_'+str(start_year)+'.tar.gz', shell=True)
 
 files = glob.glob(met_data_temporary_location+'*_'+str(start_year)+'.dat')
 w, h = 2, len(files) ;
@@ -266,15 +266,17 @@ for i,file in enumerate(files):
 # pool.close()
 
 
+print 'looping through years'
 
 for year in range(start_year,end_year+1):
+    print year
     #clean up and prexisting met files
     try:
         files_to_delete = glob.glob(met_data_temporary_location+'*.dat')
         [os.remove(f) for f in files_to_delete]
     except:
         print 'no met files to clean up'
-    subprocess.call('tar -C '+met_data_temporary_location+' -zxvf '+met_data_location+'met_data_'+str(year)+'.tar.gz', shell=True)
+    subprocess.call('tar -C '+met_data_temporary_location+' -zxf '+met_data_location+'met_data_'+str(year)+'.tar.gz', shell=True)
     try:
         shutil.move(output_directory+output_file_name+'_'+str(year), output_directory+output_file_name+'_'+str(year)+'_previous')
     except:
@@ -298,7 +300,7 @@ for year in range(start_year,end_year+1):
         [os.remove(f) for f in files_to_delete]
     except:
         print 'no met files to clean up'
-
+    pool.close()
 
 remove_files = glob.glob(base_directory+'main/*'+unique_job_id+'*')
 try: