Commit 3e856863 authored by ph290's avatar ph290
Browse files

testing improved script

parent 85d0d894
import os
import subprocess
import shutil
import glob
from math import cos, asin, sqrt
import multiprocessing as mp
from functools import partial
import uuid
##################################################
# you may need to change things here #
##################################################
base_directory = '/gpfs/ts0/home/ph290/s2p3_rv2.0/'
num_procs = mp.cpu_count() # this will use all available processors. Note that on a multi-node machine it can only use the processors on one node
# num_procs = 1
output_directory = '/gpfs/ts0/projects/Research_Project-148395/s2p3_rv2.0/output/' #where you want the output to go
output_file_name = 'test'
meterological_file_name = 'meterological_data'
domain_file_name = 's12_m2_s2_n2_h_map_test.dat'
nutrient_file_name = 'initial_nitrate_test.dat'
executable_file_name = 's2p3_rv2.0'
met_data_location = '/gpfs/ts0/projects/Research_Project-148395/s2p3_rv2.0/test/' # The location containing the tar.gz met files (in the format met_data_year.tar.gz)
met_data_temporary_location = base_directory+'met/spatial_data/test/' # The location that met data for each year will be un tar.gziped into
start_year = 2006
end_year = 2006 # same as start year resuls in a 1 year run
depth_min = 10
depth_max = 100
write_error_output = False
##################################################
# functions used by the script #
##################################################
def distance(lat1, lon1, lat2, lon2):
p = 0.017453292519943295
a = 0.5 - cos((lat2-lat1)*p)/2 + cos(lat1*p)*cos(lat2*p) * (1-cos((lon2-lon1)*p)) / 2
return 12742 * asin(sqrt(a))
def closest(data, lat1,lon1):
return min(data, key=lambda p: distance(lat1,lon1,p[0],p[1]))
# def return_domain_lon(filename,i):
# f=open(filename)
# lines=f.readlines()
# return filter(lambda a: a != '', lines[i+1].split(' '))[0:2]
#
#
f=open(base_directory+'domain/'+domain_file_name)
lines=f.readlines()
lat_domain=[]
lon_domain=[]
line_number=[]
for i,line in enumerate(lines[1::]):
depth = float(line[78:84])
if (depth >= depth_min) & (depth <= depth_max):
lon_domain.append(line[0:8])
lat_domain.append(line[8:16])
line_number.append(i)
def run_model(domain_file_name,lats_lons,year,start_year,unique_job_id,met_data_temporary_location,lon_domain,lat_domain,line_number,i):
#modifying so that the fortran code looks for the correct met file, rather than us having to copy it into the working dorectory
# lon,lat = return_domain_lon(base_directory+'domain/'+domain_file_name,i)
forcing_lat_lon = closest(lats_lons, float(lat_domain[i]),float(lon_domain[i]))
run_command = """./"""+executable_file_name+""" << EOF
"""+str(start_year)+"""
"""+str(year)+"""
"""+str(forcing_lat_lon[0])+"""
"""+str(forcing_lat_lon[1])+"""
../domain/"""+domain_file_name+"""
../domain/"""+nutrient_file_name+"""
"""+unique_job_id+"""
"""+met_data_temporary_location+"""
map
"""+str(line_number[i]+1)+"""
"""+str(start_year)+"""
"""+str(year)+"""
"""+str(forcing_lat_lon[0])+"""
"""+str(forcing_lat_lon[1])+"""
../domain/"""+domain_file_name+"""
../domain/"""+nutrient_file_name+"""
"""+unique_job_id+"""
"""+met_data_temporary_location+"""
map
"""+str(line_number[i]+1)+"""
EOF"""
# print run_command
proc = subprocess.Popen([run_command], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = proc.communicate()
# return out
return out,err
##################################################
# main program #
##################################################
unique_job_id = str(uuid.uuid4())
num_lines = sum(1 for line in open(base_directory+'domain/'+domain_file_name)) - 1
# num_lines = 10
subprocess.call('tar -C '+met_data_temporary_location+' -zxvf '+met_data_location+'met_data_'+str(start_year)+'.tar.gz', shell=True)
files = glob.glob(met_data_temporary_location+'*_'+str(start_year)+'.dat')
w, h = 2, len(files) ;
lats_lons = [[0 for x in range(w)] for y in range(h)]
for i,file in enumerate(files):
tmp = file.split('lat')[-1].split('.dat')[0].split('lon')
lats_lons[i][0] = float(tmp[0])
lats_lons[i][1] = float(tmp[1].split('_')[0])
for year in range(start_year,end_year+1):
#clean up and prexisting met files
try:
files_to_delete = glob.glob(met_data_temporary_location+'*.dat')
[os.remove(f) for f in files_to_delete]
except:
print 'no met files to clean up'
subprocess.call('tar -C '+met_data_temporary_location+' -zxvf '+met_data_location+'met_data_'+str(year)+'.tar.gz', shell=True)
try:
shutil.move(output_directory+output_file_name+'_'+str(year), output_directory+output_file_name+'_'+str(year)+'_previous')
except:
print 'no previous output file to move'
pool = mp.Pool(processes=num_procs)
func = partial(run_model, domain_file_name, lats_lons, year, start_year, unique_job_id, met_data_temporary_location,lon_domain,lat_domain,line_number)
# results,errors = pool.map(func, range(num_lines))
results, errors = zip(*pool.map(func, range(len(line_number))))
# results = pool.map(func, range(num_lines))
with open(output_directory+output_file_name+'_'+str(year),'w') as fout:
for result in results:
fout.write(result)
if write_error_output:
with open(output_directory+output_file_name+'_error_'+str(year),'w') as fout:
for error in errors:
fout.write(error)
#clean up and leftover met files
try:
files_to_delete = glob.glob(met_data_temporary_location+'*.dat')
[os.remove(f) for f in files_to_delete]
except:
print 'no met files to clean up'
remove_files = glob.glob(base_directory+'main/*'+unique_job_id+'*')
try:
remove_files.remove(base_directory+'main/restart'+unique_job_id+'.dat')
except:
pass
for remove_file in remove_files:
os.remove(remove_file)
# for year in range(start_year,end_year+1):
# pool = mp.Pool(processes=num_procs)
# func = partial(run_model2, domain_file_name, lats_lons,year,start_year,unique_job_id, met_data_temporary_location)
# # results,errors = pool.map(func, range(num_lines))
# results = pool.map(func, range(num_lines))
# with open(output_directory+output_file_name+'_'+str(year)+'_error','w') as fout:
# for result in results:
# fout.write(result)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment