diff --git a/Download_data b/Download_data deleted file mode 100644 index 155c9db..0000000 --- a/Download_data +++ /dev/null @@ -1,2 +0,0 @@ -TELECHARGER DONNEES HST -https://archive.stsci.edu/hst/search.php diff --git a/pipeline.drawio b/pipeline.drawio deleted file mode 100644 index 745b933..0000000 --- a/pipeline.drawio +++ /dev/null @@ -1,363 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/pipeline.png b/pipeline.png index 33b7fa2..a0d7d58 100644 Binary files a/pipeline.png and b/pipeline.png differ diff --git a/pipeline.svg b/pipeline.svg index 88c43d4..c41f468 100644 --- a/pipeline.svg +++ b/pipeline.svg @@ -1 +1,11287 @@ -
Raw Data
Raw Data
HST Legacy Archive
HST Legacy Archive
Pol 120
Pol 120
Pol 60
Pol 60
Pol 0
Pol 0
Uncertainty 120
Uncertainty 120
Uncertainty 60
Uncertainty 60
Uncertainty 0
Uncertainty 0
Pol 0
Pol 0
Uncertainty 0
Uncertainty 0
Pol 60
Pol 60
Uncertainty 60
Uncertainty 60
Pol 120
Pol 120
Uncertainty 120
Uncertainty 120
Pol 120
Pol 120
Pol 60
Pol 60
Pol 0
Pol 0
Uncertainty 120
Uncertainty 120
Uncertainty 60
Uncertainty 60
Uncertainty 0
Uncertainty 0
HWP 120.0
HWP 120.0
HWP 60.0
HWP 60.0
HWP 0.0
HWP 0.0
HWP 120.0
HWP 120.0
HWP 60.0
HWP 60.0
HWP 0.0
HWP 0.0
Crop Data
Crop Data
Subtract Background
Subtract Background
Align Data
Align Data
Combine Data
Combine Data
Stokes
Stokes
Stokes I
Stokes I
Stokes Q
Stokes Q
Stokes U
Stokes U
Stokes IQU covariance
Stokes IQU covariance
See Section 2.7 for computation
See Section 2.7 for computation
Compute Stoke
Compute Stoke
Rotated Stokes
Rotated Stokes
Stokes I
Stokes I
Stokes Q
Stokes Q
Stokes U
Stokes U
Stokes IQU covariance
Stokes IQU covariance
Rotate Stoke
Rotate Stoke
Polarization Parameters
Polarization Pa...
Debiased Polarization Degree
Debiased Polarization...
Polarization Angle
Polarization Angle
Polarized Flux
Polarized Flux
Polarization Degree Uncertainty
Polarization Degree U...
See Section 2.7 for computation
See Section 2.7 for computation
Compute Polarization
Compute Polarization
Compute maximal or minimal convex hull for all images
Compute maximal or min...
Compute and subtract mean value of the Gaussian-fitted intensity histogram and compute first order uncertainties
Compute and subtract m...
Aligned background subtracted images using cross-correlation and propagate uncertainties through image displacement
Aligned background sub...
Combine multiple observations through same filter with weighted gaussian smoothing
Combine multiple obser...
Rotate computed Stokes parameters and uncertainties to have North up
Rotate computed Stokes...
Compute polarization parameters from the rotated Stokes parameters and uncertainties
Compute polarization p...
Compute the Stokes parameters and uncertainties from the combined images and uncertainties and propagate polarizer axis uncertainty
Compute the Stokes par...Text is not SVG - cannot display \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/FOC_reduction.py b/src/FOC_reduction.py index f206c08..d93c1d9 100755 --- a/src/FOC_reduction.py +++ b/src/FOC_reduction.py @@ -5,117 +5,16 @@ Main script where are progressively added the steps for the FOC pipeline reducti """ #Project libraries -import sys import numpy as np from copy import deepcopy import lib.fits as proj_fits #Functions to handle fits files import lib.reduction as proj_red #Functions used in reduction pipeline import lib.plots as proj_plots #Functions for plotting data -from lib.convex_hull import image_hull from lib.deconvolve import from_file_psf -import matplotlib.pyplot as plt -from astropy.wcs import WCS - -##### User inputs -## Input and output locations -globals()['data_folder'] = "../data/NGC1068_x274020/" -globals()['infiles'] = ['x274020at_c0f.fits','x274020bt_c0f.fits','x274020ct_c0f.fits', - 'x274020dt_c0f.fits','x274020et_c0f.fits','x274020ft_c0f.fits', - 'x274020gt_c0f.fits','x274020ht_c0f.fits','x274020it_c0f.fits'] -#psf_file = 'NGC1068_f253m00.fits' -globals()['plots_folder'] = "../plots/NGC1068_x274020/" - -#globals()['data_folder'] = "../data/IC5063_x3nl030/" -#globals()['infiles'] = ['x3nl0301r_c0f.fits','x3nl0302r_c0f.fits','x3nl0303r_c0f.fits'] -##psf_file = 'IC5063_f502m00.fits' -#globals()['plots_folder'] = "../plots/IC5063_x3nl030/" - -#globals()['data_folder'] = "../data/NGC1068_x14w010/" -#globals()['infiles'] = ['x14w0101t_c0f.fits','x14w0102t_c0f.fits','x14w0103t_c0f.fits', -# 'x14w0104t_c0f.fits','x14w0105p_c0f.fits','x14w0106t_c0f.fits'] -#globals()['plots_folder'] = "../plots/NGC1068_x14w010/" - -#globals()['data_folder'] = "../data/3C405_x136060/" -#globals()['infiles'] = ['x1360601t_c0f.fits','x1360602t_c0f.fits','x1360603t_c0f.fits'] -#globals()['plots_folder'] = "../plots/3C405_x136060/" - -#globals()['data_folder'] = "../data/CygnusA_x43w0/" -#globals()['infiles'] = ['x43w0101r_c0f.fits', 'x43w0102r_c0f.fits', 'x43w0103r_c0f.fits', -# 'x43w0104r_c0f.fits', 'x43w0105r_c0f.fits', 'x43w0106r_c0f.fits', -# 'x43w0107r_c0f.fits', 'x43w0108r_c0f.fits', 'x43w0109r_c0f.fits'] #F342W -##globals()['infiles'] = ['x43w0201r_c0f.fits', 'x43w0202r_c0f.fits', 'x43w0203r_c0f.fits', -## 'x43w0204r_c0f.fits', 'x43w0205r_c0f.fits', 'x43w0206r_c0f.fits'] #F275W -#globals()['plots_folder'] = "../plots/CygnusA_x43w0/" - -#globals()['data_folder'] = "../data/3C109_x3mc010/" -#globals()['infiles'] = ['x3mc0101m_c0f.fits','x3mc0102m_c0f.fits','x3mc0103m_c0f.fits'] -#globals()['plots_folder'] = "../plots/3C109_x3mc010/" - -#globals()['data_folder'] = "../data/MKN463_x2rp030/" -#globals()['infiles'] = ['x2rp0201t_c0f.fits', 'x2rp0202t_c0f.fits', 'x2rp0203t_c0f.fits', -# 'x2rp0204t_c0f.fits', 'x2rp0205t_c0f.fits', 'x2rp0206t_c0f.fits', -# 'x2rp0207t_c0f.fits', 'x2rp0301t_c0f.fits', 'x2rp0302t_c0f.fits', -# 'x2rp0303t_c0f.fits', 'x2rp0304t_c0f.fits', 'x2rp0305t_c0f.fits', -# 'x2rp0306t_c0f.fits', 'x2rp0307t_c0f.fits'] -#globals()['plots_folder'] = "../plots/MKN463_x2rp030/" - -#globals()['data_folder'] = "../data/PG1630+377_x39510/" -#globals()['infiles'] = ['x3990201m_c0f.fits', 'x3990205m_c0f.fits', 'x3995101r_c0f.fits', -# 'x3995105r_c0f.fits', 'x3995109r_c0f.fits', 'x3995201r_c0f.fits', -# 'x3995205r_c0f.fits', 'x3990202m_c0f.fits', 'x3990206m_c0f.fits', -# 'x3995102r_c0f.fits', 'x3995106r_c0f.fits', 'x399510ar_c0f.fits', -# 'x3995202r_c0f.fits','x3995206r_c0f.fits'] -#globals()['plots_folder'] = "../plots/PG1630+377_x39510/" - -#globals()['data_folder'] = "../data/MKN3_x3nl010/" -#globals()['infiles'] = ['x3nl0101r_c0f.fits','x3nl0102r_c0f.fits','x3nl0103r_c0f.fits'] -#globals()['plots_folder'] = "../plots/MKN3_x3nl010/" - -#globals()['data_folder'] = "../data/MKN3_x3md010/" -#globals()['infiles'] = ['x3md0101r_c0f.fits', 'x3md0102r_c0f.fits', 'x3md0103r_c0f.fits'] #F275W -##globals()['infiles'] = ['x3md0104r_c0f.fits', 'x3md0105r_c0f.fits', 'x3md0106r_c0f.fits'] #F342W -#globals()['plots_folder'] = "../plots/MKN3_x3md010/" - -#globals()['data_folder'] = "../data/MKN78_x3nl020/" -#globals()['infiles'] = ['x3nl0201r_c0f.fits','x3nl0202r_c0f.fits','x3nl0203r_c0f.fits'] -#globals()['plots_folder'] = "../plots/MKN78_x3nl020/" - -#globals()['data_folder'] = "../data/MRK231_x4qr010/" -#globals()['infiles'] = ['x4qr010ar_c0f.fits', 'x4qr010br_c0f.fits', 'x4qr010dr_c0f.fits', -# 'x4qr010er_c0f.fits', 'x4qr010gr_c0f.fits', 'x4qr010hr_c0f.fits', -# 'x4qr010jr_c0f.fits', 'x4qr010kr_c0f.fits', 'x4qr0104r_c0f.fits', -# 'x4qr0105r_c0f.fits', 'x4qr0107r_c0f.fits', 'x4qr0108r_c0f.fits'] -#globals()['plots_folder'] = "../plots/MRK231_x4qr010/" - -#globals()['data_folder'] = "../data/3C273_x0u20/" -#globals()['infiles'] = ['x0u20101t_c0f.fits','x0u20102t_c0f.fits','x0u20103t_c0f.fits', -# 'x0u20104t_c0f.fits','x0u20105t_c0f.fits','x0u20106t_c0f.fits', -# 'x0u20201t_c0f.fits','x0u20202t_c0f.fits','x0u20203t_c0f.fits', -# 'x0u20204t_c0f.fits','x0u20205t_c0f.fits','x0u20206t_c0f.fits', -# 'x0u20301t_c0f.fits','x0u20302t_c0f.fits','x0u20303t_c0f.fits', -# 'x0u20304t_c0f.fits','x0u20305t_c0f.fits','x0u20306t_c0f.fits'] -#globals()['plots_folder'] = "../plots/3C273_x0u20/" - -#BEWARE: 5 observations separated by 1 year each (1995, 1996, 1997, 1998, 1999) -#globals()['data_folder'] = "../data/M87/POS1/" -##globals()['infiles'] = ['x2py010ct_c0f.fits','x2py010dt_c0f.fits','x2py010et_c0f.fits','x2py010ft_c0f.fits'] #1995 -##globals()['infiles'] = ['x3be010ct_c0f.fits','x3be010dt_c0f.fits','x3be010et_c0f.fits','x3be010ft_c0f.fits'] #1996 -##globals()['infiles'] = ['x43r010km_c0f.fits','x43r010mm_c0f.fits','x43r010om_c0f.fits','x43r010rm_c0f.fits'] #1997 -##globals()['infiles'] = ['x43r110kr_c0f.fits','x43r110mr_c0f.fits','x43r110or_c0f.fits','x43r110rr_c0f.fits'] #1998 -#globals()['infiles'] = ['x43r210kr_c0f.fits','x43r210mr_c0f.fits','x43r210or_c0f.fits','x43r210rr_c0f.fits'] #1999 -#globals()['plots_folder'] = "../plots/M87/POS1/" - -#BEWARE: 5 observations separated by 1 year each (1995, 1996, 1997, 1998, 1999) -#globals()['data_folder'] = "../data/M87/POS3/" -##globals()['infiles'] = ['x2py030at_c0f.fits','x2py030bt_c0f.fits','x2py030ct_c0f.fits','x2py0309t_c0f.fits'] #1995 -##globals()['infiles'] = ['x3be030at_c0f.fits','x3be030bt_c0f.fits','x3be030ct_c0f.fits','x3be0309t_c0f.fits'] #1996 -##globals()['infiles'] = ['x43r030em_c0f.fits','x43r030gm_c0f.fits','x43r030im_c0f.fits','x43r030lm_c0f.fits'] #1997 -##globals()['infiles'] = ['x43r130er_c0f.fits','x43r130fr_c0f.fits','x43r130ir_c0f.fits','x43r130lr_c0f.fits'] #1998 -#globals()['infiles'] = ['x43r230er_c0f.fits','x43r230fr_c0f.fits','x43r230ir_c0f.fits','x43r230lr_c0f.fits'] #1999 -#globals()['plots_folder'] = "../plots/M87/POS3/" +from lib.query import retrieve_products -def main(): +def main(target=None, proposal_id=None, infiles=None): ## Reduction parameters # Deconvolution deconvolve = False @@ -127,35 +26,41 @@ def main(): psf_shape=(25,25) iterations = 5 algo="richardson" + # Initial crop - display_crop = True - # Error estimation - error_sub_type = (51,51)#'freedman-diaconis' #sqrt, sturges, rice, scott, freedman-diaconis (default) or shape (example (51,51)) + display_crop = False + + # Background estimation + error_sub_type = 'freedman-diaconis' #sqrt, sturges, rice, scott, freedman-diaconis (default) or shape (example (51,51)) subtract_error = 1.25 - display_error = True + display_error = False + # Data binning rebin = True pxsize = 0.10 px_scale = 'arcsec' #pixel, arcsec or full rebin_operation = 'sum' #sum or average + # Alignement align_center = 'image' #If None will align image to image center - display_data = True + display_data = False + # Smoothing smoothing_function = 'combine' #gaussian_after, weighted_gaussian_after, gaussian, weighted_gaussian or combine smoothing_FWHM = 0.20 #If None, no smoothing is done smoothing_scale = 'arcsec' #pixel or arcsec + # Rotation rotate_stokes = True rotate_data = False #rotation to North convention can give erroneous results + # Final crop crop = False #Crop to desired ROI - final_display = True + final_display = False #Whether to display all polarization map outputs + # Polarization map output - figname = 'NGC1068_FOC' #target/intrument name - figtype = '_c_020' #additionnal informations - SNRp_cut = 5. #P measurments with SNR>3 - SNRi_cut = 50. #I measurments with SNR>30, which implies an uncertainty in P of 4.7%. + SNRp_cut = 3. #P measurments with SNR>3 + SNRi_cut = 30. #I measurments with SNR>30, which implies an uncertainty in P of 4.7%. vec_scale = 2.0 step_vec = 1 #plot all vectors in the array. if step_vec = 2, then every other vector will be plotted # if step_vec = 0 then all vectors are displayed at full length @@ -163,8 +68,21 @@ def main(): ##### Pipeline start ## Step 1: # Get data from fits files and translate to flux in erg/cm²/s/Angstrom. + if not infiles is None: + products = [np.array([["/".join(filepath.split('/')[:-1]),filepath.split('/')[-1]] for filepath in infiles],dtype=str)] + else: + products = retrieve_products(target,proposal_id) + data_folder = products[0][0,0] + try: + plots_folder = data_folder.replace("data","plots") + except: + plots_folder = "." + infiles = products[0][:,1] data_array, headers = proj_fits.get_obs_data(infiles, data_folder=data_folder, compute_flux=True) + figname = "_".join([target,"FOC"]) + figtype = "_".join(["".join([s[0] for s in smoothing_function.split("_")]),str(smoothing_FWHM).replace(".","")]) #additionnal informations + # Crop data to remove outside blank margins. data_array, error_array, headers = proj_red.crop_array(data_array, headers, step=5, null_val=0., inside=True, display=display_crop, savename=figname, plots_folder=plots_folder) @@ -191,11 +109,7 @@ def main(): #Plot array for checking output if display_data and px_scale.lower() not in ['full','integrate']: - vertex = image_hull(data_mask,step=5,null_val=0.,inside=True) - shape = np.array([vertex[1]-vertex[0],vertex[3]-vertex[2]]) - rectangle = [vertex[2], vertex[0], shape[1], shape[0], 0., 'g'] - - proj_plots.plot_obs(data_array, headers, vmin=data_array[data_array>0.].min()*headers[0]['photflam'], vmax=data_array[data_array>0.].max()*headers[0]['photflam'], rectangle =[rectangle,]*data_array.shape[0], savename=figname+"_center_"+align_center, plots_folder=plots_folder) + proj_plots.plot_obs(data_array, headers, vmin=data_array[data_array>0.].min()*headers[0]['photflam'], vmax=data_array[data_array>0.].max()*headers[0]['photflam'], savename=figname+"_center_"+align_center, plots_folder=plots_folder) background = np.array([np.array(bkg).reshape(1,1) for bkg in background]) background_error = np.array([np.array(np.sqrt((bkg-background[np.array([h['filtnam1']==head['filtnam1'] for h in headers],dtype=bool)].mean())**2/np.sum([h['filtnam1']==head['filtnam1'] for h in headers]))).reshape(1,1) for bkg,head in zip(background,headers)]) @@ -258,5 +172,17 @@ def main(): return 0 + if __name__ == "__main__": - sys.exit(main()) + import argparse + + parser = argparse.ArgumentParser(description='Query MAST for target products') + parser.add_argument('-t','--target', metavar='targetname', required=False, + help='the name of the target', type=str, default=None) + parser.add_argument('-p','--proposal_id', metavar='proposal_id', required=False, + help='the proposal id of the data products', type=int, default=None) + parser.add_argument('-f','--files', metavar='path', required=False, nargs='*', + help='the full or relative path to the data products', default=None) + args = parser.parse_args() + prodpaths = main(target=args.target, proposal_id=args.proposal_id, infiles=args.files) + print(prodpaths) \ No newline at end of file diff --git a/src/comparison_Kishimoto.py b/src/comparison_Kishimoto.py index bbb504e..b81edb5 100755 --- a/src/comparison_Kishimoto.py +++ b/src/comparison_Kishimoto.py @@ -19,6 +19,7 @@ root_dir_S = path_join(root_dir,'FOC_Reduction','output') root_dir_data_S = path_join(root_dir,'FOC_Reduction','data','NGC1068_x274020') root_dir_plot_S = path_join(root_dir,'FOC_Reduction','plots','NGC1068_x274020') filename_S = "NGC1068_FOC_b_10px.fits" +plt.rcParams.update({'font.size': 15}) data_K = {} data_S = {} @@ -107,7 +108,6 @@ fig_pa.savefig(path_join(root_dir_plot_S,"NGC1068_K_pol_ang.png"),bbox_inches="t ##### ###display both polarization maps to check consistency ##### -plt.rcParams.update({'font.size': 10}) fig = plt.figure(num="Polarization maps comparison") ax = fig.add_subplot(111, projection=wcs) fig.subplots_adjust(right=0.85) @@ -132,7 +132,7 @@ ax.coords[1].set_ticklabel_position('l') #ax.axis('equal') cbar = plt.colorbar(im0, cax=cbar_ax, label=r"$F_{\lambda}$ [$ergs \cdot cm^{-2} \cdot s^{-1} \cdot \AA^{-1}$]") -plt.rcParams.update({'font.size': 8}) +#plt.rcParams.update({'font.size': 8}) ax.legend(loc='upper right') fig.savefig(path_join(root_dir_plot_S,"NGC1068_K_comparison.png"),bbox_inches="tight") diff --git a/src/lib/background.py b/src/lib/background.py index edce04c..1c175b9 100644 --- a/src/lib/background.py +++ b/src/lib/background.py @@ -10,6 +10,7 @@ prototypes : Compute the error (noise) of the input array by looking at the sub-region of minimal flux in every image and of shape sub_shape. """ import sys +from os.path import join as path_join from copy import deepcopy import numpy as np import matplotlib.pyplot as plt @@ -59,7 +60,7 @@ def display_bkg(data, background, std_bkg, headers, histograms=None, binning=Non ax.set_ylabel(r"Flux [$ergs \cdot cm^{-2} \cdot s^{-1} \cdot \AA^{-1}$]") plt.legend() if not(savename is None): - fig.savefig(plots_folder+savename+"_background_flux.png", bbox_inches='tight') + fig.savefig(path_join(plots_folder,savename+"_background_flux.png"), bbox_inches='tight') if not(histograms is None): filt_obs = {"POL0":0, "POL60":0, "POL120":0} @@ -78,7 +79,7 @@ def display_bkg(data, background, std_bkg, headers, histograms=None, binning=Non ax_h.set_title("Histogram for each observation") plt.legend() if not(savename is None): - fig_h.savefig(plots_folder+savename+'_histograms.png', bbox_inches='tight') + fig_h.savefig(path_join(plots_folder,savename+'_histograms.png'), bbox_inches='tight') fig2, ax2 = plt.subplots(figsize=(10,10)) data0 = data[0]*convert_flux[0] @@ -103,7 +104,7 @@ def display_bkg(data, background, std_bkg, headers, histograms=None, binning=Non fig2.colorbar(im, cax=cbar_ax, label=r"Flux [$ergs \cdot cm^{-2} \cdot s^{-1} \cdot \AA^{-1}$]") if not(savename is None): - fig2.savefig(plots_folder+savename+'_'+filt+'_background_location.png', bbox_inches='tight') + fig2.savefig(path_join(plots_folder,savename+'_'+filt+'_background_location.png'), bbox_inches='tight') if not(rectangle is None): plot_obs(data, headers, vmin=data[data > 0.].min()*convert_flux.mean(), vmax=data[data > 0.].max()*convert_flux.mean(), rectangle=rectangle, savename=savename+"_background_location",plots_folder=plots_folder) diff --git a/src/lib/fits.py b/src/lib/fits.py index 6d6146f..97fc9ec 100755 --- a/src/lib/fits.py +++ b/src/lib/fits.py @@ -10,6 +10,7 @@ prototypes : """ import numpy as np +from os.path import join as path_join from astropy.io import fits from astropy import wcs from lib.convex_hull import image_hull, clean_ROI @@ -39,7 +40,7 @@ def get_obs_data(infiles, data_folder="", compute_flux=False): """ data_array, headers = [], [] for i in range(len(infiles)): - with fits.open(data_folder+infiles[i]) as f: + with fits.open(path_join(data_folder,infiles[i])) as f: headers.append(f[0].header) data_array.append(f[0].data) data_array = np.array(data_array,dtype=np.double) @@ -188,7 +189,7 @@ def save_Stokes(I_stokes, Q_stokes, U_stokes, Stokes_cov, P, debiased_P, s_P, hdul.append(hdu) #Save fits file to designated filepath - hdul.writeto(data_folder+filename+".fits", overwrite=True) + hdul.writeto(path_join(data_folder,filename+".fits"), overwrite=True) if return_hdul: return hdul diff --git a/src/lib/plots.py b/src/lib/plots.py index 6761057..89621f9 100755 --- a/src/lib/plots.py +++ b/src/lib/plots.py @@ -38,6 +38,7 @@ prototypes : from copy import deepcopy import numpy as np +from os.path import join as path_join import matplotlib.pyplot as plt from matplotlib.patches import Rectangle, Circle from matplotlib.path import Path @@ -161,7 +162,7 @@ def plot_obs(data_array, headers, shape=None, vmin=None, vmax=None, rectangle=No if not (savename is None): #fig.suptitle(savename) - fig.savefig(plots_folder+savename+".png",bbox_inches='tight') + fig.savefig(path_join(plots_folder,savename+".png"),bbox_inches='tight') plt.show() return 0 @@ -211,7 +212,7 @@ def plot_Stokes(Stokes, savename=None, plots_folder=""): if not (savename is None): #fig.suptitle(savename+"_IQU") - fig.savefig(plots_folder+savename+"_IQU.png",bbox_inches='tight') + fig.savefig(path_join(plots_folder,savename+"_IQU.png"),bbox_inches='tight') plt.show() return 0 @@ -320,7 +321,7 @@ def polarization_map(Stokes, data_mask=None, rectangle=None, SNRp_cut=3., SNRi_c if display.lower() in ['intensity']: # If no display selected, show intensity map display='i' - vmin, vmax = 1.*np.mean(np.sqrt(stk_cov.data[0,0][mask])*convert_flux), np.max(stkI.data[stkI.data > 0.]*convert_flux) + vmin, vmax = 1/5.0*np.mean(np.sqrt(stk_cov.data[0,0][mask])*convert_flux), np.max(stkI.data[stkI.data > 0.]*convert_flux) im = ax.imshow(stkI.data*convert_flux, norm=LogNorm(vmin,vmax), aspect='equal', cmap='inferno', alpha=1.) cbar = plt.colorbar(im, cax=cbar_ax, label=r"$F_{\lambda}$ [$ergs \cdot cm^{-2} \cdot s^{-1} \cdot \AA^{-1}$]") levelsI = np.linspace(vmax*0.01, vmax*0.99, 10) @@ -446,7 +447,7 @@ def polarization_map(Stokes, data_mask=None, rectangle=None, SNRp_cut=3., SNRi_c if not savename is None: #fig.suptitle(savename) - fig.savefig(plots_folder+savename+".png",bbox_inches='tight',dpi=300) + fig.savefig(path_join(plots_folder,savename+".png"),bbox_inches='tight',dpi=300) plt.show() return fig, ax @@ -1766,7 +1767,7 @@ class pol_map(object): self.display_selection = "total_flux" if self.display_selection.lower() in ['total_flux']: self.data = self.I*self.convert_flux - vmin, vmax = 1/2.0*np.median(self.data[self.data > 0.]), np.max(self.data[self.data > 0.]) + vmin, vmax = 1/5.0*np.median(self.data[self.data > 0.]), np.max(self.data[self.data > 0.]) norm = LogNorm(vmin, vmax) label = r"$F_{\lambda}$ [$ergs \cdot cm^{-2} \cdot s^{-1} \cdot \AA^{-1}$]" elif self.display_selection.lower() in ['pol_flux']: diff --git a/src/lib/query.py b/src/lib/query.py index c78d4bc..d0133ca 100644 --- a/src/lib/query.py +++ b/src/lib/query.py @@ -1,5 +1,3 @@ -from sys import exit as sys_exit, argv as sys_argv -from getopt import getopt, error as geterr from os import system from os.path import join as path_join, exists as path_exists from astroquery.mast import MastMissions, Observations @@ -102,10 +100,11 @@ def get_product_list(target, proposal_id=None): return products -def retrieve_products(target, products): +def retrieve_products(target, proposal_id=None): """ - Given a target name and a products list, create the local directories and retrieve the fits files from the MAST Archive + Given a target name and a proposal_id, create the local directories and retrieve the fits files from the MAST Archive """ + products = get_product_list(target=target,proposal_id=proposal_id) prodpaths = [] data_dir = path_join("../data", target) out = "" @@ -113,7 +112,7 @@ def retrieve_products(target, products): filepaths = [] obs_dir = path_join(data_dir, obs_id) if not path_exists(obs_dir): - system("mkdir -p "+obs_dir) + system("mkdir -p {0:s} {1:s}".format(obs_dir,path_join("../plots",target,obs_id))) for file in products['productFilename'][products['proposal_id'] == obs_id]: fpath = path_join(obs_dir, file) if not path_exists(fpath): @@ -121,37 +120,20 @@ def retrieve_products(target, products): products['dataURI'][products['productFilename'] == file][0], local_path=fpath)[0]) else: out += "{0:s} : Exists\n".format(file) - filepaths.append(fpath) - prodpaths.append(filepaths) + filepaths.append([obs_dir,file]) + prodpaths.append(np.array(filepaths,dtype=str)) return prodpaths -def main(): - - target, proposal_id = None, None - try: - arg, _ = getopt(sys_argv[1:], "ht:p:", ["help", "target=", "proposal_id="]) - for curr_arg, curr_val in arg: - if curr_arg in ("-h", "--help"): - print("python3 query.py -t -p ") - elif curr_arg in ("-t", "--target"): - target = str(curr_val) - elif curr_arg in ("-p", "--proposal_id"): - proposal_id = int(curr_val) - except geterr: - print(str(geterr)) - - if target is None: - target = input("Target name\n>") - - products = get_product_list(target=target,proposal_id=proposal_id) - products_paths = retrieve_products(target=target, products=products) - - print(products_paths) - - return 0 - - if __name__ == "__main__": - sys_exit(main()) + import argparse + + parser = argparse.ArgumentParser(description='Query MAST for target products') + parser.add_argument('-t','--target', metavar='targetname', required=False, + help='the name of the target', type=str, default=None) + parser.add_argument('-p','--proposal_id', metavar='proposal_id', required=False, + help='the proposal id of the data products', type=int, default=None) + args = parser.parse_args() + prodpaths = retrieve_products(target=args.target, proposal_id=args.proposal_id) + print(prodpaths) \ No newline at end of file diff --git a/src/run_pipeline b/src/run_pipeline deleted file mode 100644 index 6c943bf..0000000 --- a/src/run_pipeline +++ /dev/null @@ -1,8 +0,0 @@ -EXECUTER PIPELINE -python3 FOC_reduction.py - -OUVRIR LE FICHIER FITS FINAL AVEC L'OUTIL D'ANALYSE -python3 analysis.py -f ../data/NGC1068_x274020/NGC1068_FOC_combine_FWHM010.fits - -METTRE DES VALEURS PAR DEFAUT A L'OUTIL D'ANALYSE POUR LES CUTS EN POLARIZATION ET INTENSITE -python3 analysis.py -f ../data/NGC1068_x274020/NGC1068_FOC_combine_FWHM010.fits -p 5 -i 10 diff --git a/update_git b/update_git deleted file mode 100644 index 7b3950d..0000000 --- a/update_git +++ /dev/null @@ -1,14 +0,0 @@ -RECUPERER NOUVELLE VERSION -git fetch - -INSTALLER NOUVELLE VERSION -git pull origin main - -AJOUTER TOUS LES FICHIERS A L'ARBRE DE DIFFERENTIATION DU GIT -git add . - -PRENDRE TOUS LES FICHIERS DE L'ARBRE COURANT POUR EN FAIRE UNE VERSION -git commit -m "mesage du commit" - -ENVOYER MES COMMITS LOCAUX AU DOSSIER PARTAGE DISTANT -git push -u origin main