Commit 7b49b8c5 authored by ph290's avatar ph290
Browse files

making sure that parallel jobs are cleaned up after running

parent 291a0dd6
...@@ -26,7 +26,7 @@ met_data_location = '/gpfs/ts0/projects/Research_Project-148395/s2p3_rv2.0/test/ ...@@ -26,7 +26,7 @@ met_data_location = '/gpfs/ts0/projects/Research_Project-148395/s2p3_rv2.0/test/
met_data_temporary_location = base_directory+'met/spatial_data/' # The location that met data for each year will be un tar.gziped into met_data_temporary_location = base_directory+'met/spatial_data/' # The location that met data for each year will be un tar.gziped into
start_year = 2006 start_year = 2006
end_year = 2006 # same as start year resuls in a 1 year run end_year = 2006 # same as start year resuls in a 1 year run
depth_min = 10 # NOTE that these numbers MUST be the same as those used in the scripts used to produce the meterology and nutrient files, otherwse data will not be taken for teh correct lats/lons and/or the script will fail depth_min = 10 # NOTE that these numbers MUST be the same as those used in the scripts used to produce the meterology and nutrient files, otherwse data will not be taken for teh correct lats/lons and/or the script will fail
depth_max = 100 depth_max = 100
write_error_output = False write_error_output = False
...@@ -130,7 +130,7 @@ def run_model(domain_file_name,lats_lons,year,start_year,unique_job_id,met_data_ ...@@ -130,7 +130,7 @@ def run_model(domain_file_name,lats_lons,year,start_year,unique_job_id,met_data_
if lon_domain_tmp < 0.0: if lon_domain_tmp < 0.0:
lon_domain_tmp = 360.0+lon_domain_tmp lon_domain_tmp = 360.0+lon_domain_tmp
# forcing_lat_lon = closest(lats_lons, float(lat_domain[i]),float(lon_domain[i])) # forcing_lat_lon = closest(lats_lons, float(lat_domain[i]),float(lon_domain[i]))
print i # print i
# print str(forcing_lat_lon[0]) # print str(forcing_lat_lon[0])
# print str(forcing_lat_lon[1]) # print str(forcing_lat_lon[1])
# print str(float(lat_domain[i])) # print str(float(lat_domain[i]))
...@@ -241,8 +241,8 @@ unique_job_id = str(uuid.uuid4()) ...@@ -241,8 +241,8 @@ unique_job_id = str(uuid.uuid4())
num_lines = sum(1 for line in open(base_directory+'domain/'+domain_file_name)) - 1 num_lines = sum(1 for line in open(base_directory+'domain/'+domain_file_name)) - 1
# num_lines = 10 # num_lines = 10
print 'initial unzipping of met data to extrcat lat and lon data'
subprocess.call('tar -C '+met_data_temporary_location+' -zxvf '+met_data_location+'met_data_'+str(start_year)+'.tar.gz', shell=True) subprocess.call('tar -C '+met_data_temporary_location+' -zxf '+met_data_location+'met_data_'+str(start_year)+'.tar.gz', shell=True)
files = glob.glob(met_data_temporary_location+'*_'+str(start_year)+'.dat') files = glob.glob(met_data_temporary_location+'*_'+str(start_year)+'.dat')
w, h = 2, len(files) ; w, h = 2, len(files) ;
...@@ -266,15 +266,17 @@ for i,file in enumerate(files): ...@@ -266,15 +266,17 @@ for i,file in enumerate(files):
# pool.close() # pool.close()
print 'looping through years'
for year in range(start_year,end_year+1): for year in range(start_year,end_year+1):
print year
#clean up and prexisting met files #clean up and prexisting met files
try: try:
files_to_delete = glob.glob(met_data_temporary_location+'*.dat') files_to_delete = glob.glob(met_data_temporary_location+'*.dat')
[os.remove(f) for f in files_to_delete] [os.remove(f) for f in files_to_delete]
except: except:
print 'no met files to clean up' print 'no met files to clean up'
subprocess.call('tar -C '+met_data_temporary_location+' -zxvf '+met_data_location+'met_data_'+str(year)+'.tar.gz', shell=True) subprocess.call('tar -C '+met_data_temporary_location+' -zxf '+met_data_location+'met_data_'+str(year)+'.tar.gz', shell=True)
try: try:
shutil.move(output_directory+output_file_name+'_'+str(year), output_directory+output_file_name+'_'+str(year)+'_previous') shutil.move(output_directory+output_file_name+'_'+str(year), output_directory+output_file_name+'_'+str(year)+'_previous')
except: except:
...@@ -298,7 +300,7 @@ for year in range(start_year,end_year+1): ...@@ -298,7 +300,7 @@ for year in range(start_year,end_year+1):
[os.remove(f) for f in files_to_delete] [os.remove(f) for f in files_to_delete]
except: except:
print 'no met files to clean up' print 'no met files to clean up'
pool.close()
remove_files = glob.glob(base_directory+'main/*'+unique_job_id+'*') remove_files = glob.glob(base_directory+'main/*'+unique_job_id+'*')
try: try:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment