tidying up

main/run_map_parallel_perhaps_faster.py

-import os
-import subprocess
-import shutil
-import glob
-from math import cos, asin, sqrt
-import multiprocessing as mp
-from functools import partial
-import uuid
-
-
-##################################################
-# you may need to change things here             #
-##################################################
-
-base_directory = '/gpfs/ts0/home/ph290/s2p3_rv2.0/'
-num_procs = mp.cpu_count() # this will use all available processors. Note that on a multi-node machine it can only use the processors on one node
-# num_procs = 1
-output_directory = '/gpfs/ts0/projects/Research_Project-148395/s2p3_rv2.0/output/'  #where you want the output to go
-output_file_name = 'test'
-meterological_file_name = 'meterological_data'
-domain_file_name = 's12_m2_s2_n2_h_map_test.dat'
-nutrient_file_name = 'initial_nitrate_test.dat'
-executable_file_name = 's2p3_rv2.0'
-met_data_location = '/gpfs/ts0/projects/Research_Project-148395/s2p3_rv2.0/test/' # The location containing the tar.gz met files (in the format met_data_year.tar.gz)
-met_data_temporary_location = base_directory+'met/spatial_data/test/' # The location that met data for each year will be un tar.gziped into
-start_year = 2006
-end_year = 2006 # same as start year resuls in a 1 year run
-depth_min = 10
-depth_max = 100
-write_error_output = False
-
-##################################################
-# functions used by the script                   #
-##################################################
-
-def distance(lat1, lon1, lat2, lon2):
-    p = 0.017453292519943295
-    a = 0.5 - cos((lat2-lat1)*p)/2 + cos(lat1*p)*cos(lat2*p) * (1-cos((lon2-lon1)*p)) / 2
-    return 12742 * asin(sqrt(a))
-
-def closest(data, lat1,lon1):
-    return min(data, key=lambda p: distance(lat1,lon1,p[0],p[1]))
-
-# def return_domain_lon(filename,i):
-#     f=open(filename)
-#     lines=f.readlines()
-#     return filter(lambda a: a != '', lines[i+1].split(' '))[0:2]
-#
-#
-
-
-f=open(base_directory+'domain/'+domain_file_name)
-lines=f.readlines()
-lat_domain=[]
-lon_domain=[]
-line_number=[]
-for i,line in enumerate(lines[1::]):
-    depth = float(line[78:84])
-    if (depth >= depth_min) & (depth <= depth_max):
-        lon_domain.append(line[0:8])
-        lat_domain.append(line[8:16])
-        line_number.append(i)
-
-
-
-
-
-
-
-def run_model(domain_file_name,lats_lons,year,start_year,unique_job_id,met_data_temporary_location,lon_domain,lat_domain,line_number,i):
-    #modifying so that the fortran code looks for the correct met file, rather than us having to copy it into the working dorectory
-    # lon,lat = return_domain_lon(base_directory+'domain/'+domain_file_name,i)
-    forcing_lat_lon = closest(lats_lons, float(lat_domain[i]),float(lon_domain[i]))
-    run_command = """./"""+executable_file_name+""" << EOF
-"""+str(start_year)+"""
-"""+str(year)+"""
-"""+str(forcing_lat_lon[0])+"""
-"""+str(forcing_lat_lon[1])+"""
-../domain/"""+domain_file_name+"""
-../domain/"""+nutrient_file_name+"""
-"""+unique_job_id+"""
-"""+met_data_temporary_location+"""
-map
-"""+str(line_number[i]+1)+"""
-"""+str(start_year)+"""
-"""+str(year)+"""
-"""+str(forcing_lat_lon[0])+"""
-"""+str(forcing_lat_lon[1])+"""
-../domain/"""+domain_file_name+"""
-../domain/"""+nutrient_file_name+"""
-"""+unique_job_id+"""
-"""+met_data_temporary_location+"""
-map
-"""+str(line_number[i]+1)+"""
-EOF"""
-    # print run_command
-    proc = subprocess.Popen([run_command],  shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    out, err = proc.communicate()
-    # return out
-    return out,err
-
-
-##################################################
-# main program                                   #
-##################################################
-
-unique_job_id = str(uuid.uuid4())
-
-num_lines = sum(1 for line in open(base_directory+'domain/'+domain_file_name)) - 1
-# num_lines = 10
-
-
-subprocess.call('tar -C '+met_data_temporary_location+' -zxvf '+met_data_location+'met_data_'+str(start_year)+'.tar.gz', shell=True)
-
-files = glob.glob(met_data_temporary_location+'*_'+str(start_year)+'.dat')
-w, h = 2, len(files) ;
-lats_lons = [[0 for x in range(w)] for y in range(h)]
-for i,file in enumerate(files):
-    tmp = file.split('lat')[-1].split('.dat')[0].split('lon')
-    lats_lons[i][0] = float(tmp[0])
-    lats_lons[i][1] = float(tmp[1].split('_')[0])
-
-
-for year in range(start_year,end_year+1):
-    #clean up and prexisting met files
-    try:
-        files_to_delete = glob.glob(met_data_temporary_location+'*.dat')
-        [os.remove(f) for f in files_to_delete]
-    except:
-        print 'no met files to clean up'
-    subprocess.call('tar -C '+met_data_temporary_location+' -zxvf '+met_data_location+'met_data_'+str(year)+'.tar.gz', shell=True)
-    try:
-        shutil.move(output_directory+output_file_name+'_'+str(year), output_directory+output_file_name+'_'+str(year)+'_previous')
-    except:
-        print 'no previous output file to move'
-    pool = mp.Pool(processes=num_procs)
-    func = partial(run_model, domain_file_name, lats_lons, year, start_year, unique_job_id, met_data_temporary_location,lon_domain,lat_domain,line_number)
-    # results,errors = pool.map(func, range(num_lines))
-    results, errors = zip(*pool.map(func, range(len(line_number))))
-    # results = pool.map(func, range(num_lines))
-    with open(output_directory+output_file_name+'_'+str(year),'w') as fout:
-        for result in results:
-            fout.write(result)
-    if write_error_output:
-        with open(output_directory+output_file_name+'_error_'+str(year),'w') as fout:
-            for error in errors:
-                fout.write(error)
-    #clean up and leftover met files
-    try:
-        files_to_delete = glob.glob(met_data_temporary_location+'*.dat')
-        [os.remove(f) for f in files_to_delete]
-    except:
-        print 'no met files to clean up'
-
-
-remove_files = glob.glob(base_directory+'main/*'+unique_job_id+'*')
-try:
-    remove_files.remove(base_directory+'main/restart'+unique_job_id+'.dat')
-except:
-    pass
-for remove_file in remove_files:
-    os.remove(remove_file)
-
-
-# for year in range(start_year,end_year+1):
-#     pool = mp.Pool(processes=num_procs)
-#     func = partial(run_model2, domain_file_name, lats_lons,year,start_year,unique_job_id, met_data_temporary_location)
-#     # results,errors = pool.map(func, range(num_lines))
-#     results = pool.map(func, range(num_lines))
-#     with open(output_directory+output_file_name+'_'+str(year)+'_error','w') as fout:
-#         for result in results:
-#             fout.write(result)