Tibeuleu/FOC_Reduction
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add subtract_bkg funcition

Browse Source 
Allow subtracting the bkg simpler
This commit is contained in:
sugar_jo
2024-07-14 15:46:22 +08:00
parent a4e8f51c50
commit 8e5f439259
4 changed files with 86 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
package/FOC_reduction.py
View File

@@ -68,6 +68,18 @@ def main(target=None, proposal_id=None, infiles=None, output_dir="./data", crop=
vec_scale = 5
step_vec = 1 # plot all vectors in the array. if step_vec = 2, then every other vector will be plotted if step_vec = 0 then all vectors are displayed at full length
# Adaptive binning
# in order to perfrom optimal binning, there are several steps to follow:
# 1. Load the data again and preserve the full images
# 2. Skip the cropping step but use the same error and background estimation
# 3. Use the same alignment as the routine
# 4. Skip the rebinning step
# 5. Calulate the Stokes parameters without smoothing
#
optimal_binning = False
optimize = False
options = {'optimize': optimize, 'optimal_binning': optimal_binning}
# Pipeline start
# Step 1:
# Get data from fits files and translate to flux in erg/cm²/s/Angstrom.
@@ -94,6 +106,9 @@ def main(target=None, proposal_id=None, infiles=None, output_dir="./data", crop=
infiles = [p[1] for p in prod]
data_array, headers = proj_fits.get_obs_data(infiles, data_folder=data_folder, compute_flux=True)
if optimal_binning:
_data_array, _headers = deepcopy(data_array), deepcopy(headers)
figname = "_".join([target, "FOC"])
figtype = ""
if rebin:
@@ -103,7 +118,7 @@ def main(target=None, proposal_id=None, infiles=None, output_dir="./data", crop=
figtype = "full"
if smoothing_FWHM is not None:
figtype += "_"+"".join(["".join([s[0] for s in smoothing_function.split("_")]),
"{0:.2f}".format(smoothing_FWHM), smoothing_scale]) # additionnal informations
"{0:.2f}".format(smoothing_FWHM), smoothing_scale]) # additionnal informations
if deconvolve:
figtype += "_deconv"
if align_center is None:
@@ -111,9 +126,12 @@ def main(target=None, proposal_id=None, infiles=None, output_dir="./data", crop=
# Crop data to remove outside blank margins.
data_array, error_array, headers = proj_red.crop_array(data_array, headers, step=5, null_val=0.,
inside=True, display=display_crop, savename=figname, plots_folder=plots_folder)
inside=True, display=display_crop, savename=figname, plots_folder=plots_folder)
data_mask = np.ones(data_array[0].shape, dtype=bool)
if optimal_binning:
_data_mask = np.ones(_data_array[0].shape, dtype=bool)
# Deconvolve data using Richardson-Lucy iterative algorithm with a gaussian PSF of given FWHM.
if deconvolve:
data_array = proj_red.deconvolve_array(data_array, headers, psf=psf, FWHM=psf_FWHM, scale=psf_scale, shape=psf_shape, iterations=iterations, algo=algo)
@@ -122,10 +140,17 @@ def main(target=None, proposal_id=None, infiles=None, output_dir="./data", crop=
background = None
data_array, error_array, headers, background = proj_red.get_error(data_array, headers, error_array, data_mask=data_mask, sub_type=error_sub_type, subtract_error=subtract_error, display=display_bkg, savename="_".join([figname, "errors"]), plots_folder=plots_folder, return_background=True)
# if optimal_binning:
# _data_array, _error_array, _background = proj_red.subtract_bkg(_data_array, error_array, background) # _background is the same as background, but for the optimal binning to clarify
# Align and rescale images with oversampling.
data_array, error_array, headers, data_mask, shifts, error_shifts = proj_red.align_data(
data_array, headers, error_array=error_array, background=background, upsample_factor=10, ref_center=align_center, return_shifts=True)
# if optimal_binning:
# _data_array, _error_array, _headers, _data_mask, _shifts, _error_shifts = proj_red.align_data(
# _data_array, _headers, error_array=_error_array, background=background, upsample_factor=10, ref_center=align_center, return_shifts=True)
if display_align:
print("Image shifts: {} \nShifts uncertainty: {}".format(shifts, error_shifts))
proj_plots.plot_obs(data_array, headers, savename="_".join([figname, str(align_center)]), plots_folder=plots_folder, norm=LogNorm(
@@ -162,6 +187,12 @@ def main(target=None, proposal_id=None, infiles=None, output_dir="./data", crop=
I_bkg, Q_bkg, U_bkg, S_cov_bkg = proj_red.compute_Stokes(background, background_error, np.array(True).reshape(
1, 1), headers, FWHM=None, scale=smoothing_scale, smoothing=smoothing_function, transmitcorr=False)
# if optimal_binning:
# _I_stokes, _Q_stokes, _U_stokes, _Stokes_cov = proj_red.compute_Stokes(
# _data_array, _error_array, _data_mask, _headers, FWHM=None, scale=smoothing_scale, smoothing=smoothing_function, transmitcorr=transmitcorr)
# _I_bkg, _Q_bkg, _U_bkg, _S_cov_bkg = proj_red.compute_Stokes(_background, background_error, np.array(True).reshape(
# 1, 1), _headers, FWHM=None, scale=smoothing_scale, smoothing=smoothing_function, transmitcorr=False)
# Step 3:
# Rotate images to have North up
if rotate_stokes:
@@ -201,26 +232,26 @@ def main(target=None, proposal_id=None, infiles=None, output_dir="./data", crop=
# Plot polarization map (Background is either total Flux, Polarization degree or Polarization degree error).
if px_scale.lower() not in ['full', 'integrate'] and not interactive:
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim,
step_vec=step_vec, vec_scale=vec_scale, savename="_".join([figname]), plots_folder=plots_folder)
step_vec=step_vec, vec_scale=vec_scale, savename="_".join([figname]), plots_folder=plots_folder, **options)
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim, step_vec=step_vec,
vec_scale=vec_scale, savename="_".join([figname, "I"]), plots_folder=plots_folder, display='Intensity')
vec_scale=vec_scale, savename="_".join([figname, "I"]), plots_folder=plots_folder, display='Intensity', **options)
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim, step_vec=step_vec,
vec_scale=vec_scale, savename="_".join([figname, "P_flux"]), plots_folder=plots_folder, display='Pol_Flux')
vec_scale=vec_scale, savename="_".join([figname, "P_flux"]), plots_folder=plots_folder, display='Pol_Flux', **options)
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim, step_vec=step_vec,
vec_scale=vec_scale, savename="_".join([figname, "P"]), plots_folder=plots_folder, display='Pol_deg')
vec_scale=vec_scale, savename="_".join([figname, "P"]), plots_folder=plots_folder, display='Pol_deg', **options)
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim, step_vec=step_vec,
vec_scale=vec_scale, savename="_".join([figname, "PA"]), plots_folder=plots_folder, display='Pol_ang')
vec_scale=vec_scale, savename="_".join([figname, "PA"]), plots_folder=plots_folder, display='Pol_ang', **options)
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim, step_vec=step_vec,
vec_scale=vec_scale, savename="_".join([figname, "I_err"]), plots_folder=plots_folder, display='I_err')
vec_scale=vec_scale, savename="_".join([figname, "I_err"]), plots_folder=plots_folder, display='I_err', **options)
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim, step_vec=step_vec,
vec_scale=vec_scale, savename="_".join([figname, "P_err"]), plots_folder=plots_folder, display='Pol_deg_err')
vec_scale=vec_scale, savename="_".join([figname, "P_err"]), plots_folder=plots_folder, display='Pol_deg_err', **options)
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim, step_vec=step_vec,
vec_scale=vec_scale, savename="_".join([figname, "SNRi"]), plots_folder=plots_folder, display='SNRi')
vec_scale=vec_scale, savename="_".join([figname, "SNRi"]), plots_folder=plots_folder, display='SNRi', **options)
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim, step_vec=step_vec,
vec_scale=vec_scale, savename="_".join([figname, "SNRp"]), plots_folder=plots_folder, display='SNRp')
vec_scale=vec_scale, savename="_".join([figname, "SNRp"]), plots_folder=plots_folder, display='SNRp', **options)
elif not interactive:
proj_plots.polarization_map(deepcopy(Stokes_test), data_mask, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut,
savename=figname, plots_folder=plots_folder, display='integrate')
savename=figname, plots_folder=plots_folder, display='integrate', **options)
elif px_scale.lower() not in ['full', 'integrate']:
proj_plots.pol_map(Stokes_test, SNRp_cut=SNRp_cut, SNRi_cut=SNRi_cut, flux_lim=flux_lim)

51
package/lib/background.py
View File

@@ -239,19 +239,22 @@ def bkg_fit(data, error, mask, headers, subtract_error=True, display=False, save
error_bkg[i] *= bkg
n_error_array[i] = np.sqrt(n_error_array[i]**2 + error_bkg[i]**2)
# n_error_array[i] = np.sqrt(n_error_array[i]**2 + error_bkg[i]**2)
# Substract background
if subtract_error > 0:
n_data_array[i][mask] = n_data_array[i][mask] - bkg
n_data_array[i][np.logical_and(mask, n_data_array[i] <= 1e-3*bkg)] = 1e-3*bkg
# if subtract_error > 0:
# n_data_array[i][mask] = n_data_array[i][mask] - bkg
# n_data_array[i][np.logical_and(mask, n_data_array[i] <= 1e-3*bkg)] = 1e-3*bkg
std_bkg[i] = image[np.abs(image-bkg)/bkg < 1.].std()
background[i] = bkg
if subtract_error > 0:
n_data_array, n_error_array, background, error_bkg = subtract_bkg(n_data_array, n_error_array, mask, background, error_bkg)
if display:
display_bkg(data, background, std_bkg, headers, histograms=histograms, binning=binning, coeff=coeff, savename=savename, plots_folder=plots_folder)
return n_data_array, n_error_array, headers, background
return n_data_array, n_error_array, headers, background, error_bkg
def bkg_hist(data, error, mask, headers, sub_type=None, subtract_error=True, display=False, savename=None, plots_folder=""):
@@ -343,19 +346,22 @@ def bkg_hist(data, error, mask, headers, sub_type=None, subtract_error=True, dis
error_bkg[i] *= bkg
n_error_array[i] = np.sqrt(n_error_array[i]**2 + error_bkg[i]**2)
# n_error_array[i] = np.sqrt(n_error_array[i]**2 + error_bkg[i]**2)
# Substract background
if subtract_error > 0:
n_data_array[i][mask] = n_data_array[i][mask] - bkg
n_data_array[i][np.logical_and(mask, n_data_array[i] <= 1e-3*bkg)] = 1e-3*bkg
# if subtract_error > 0:
# n_data_array[i][mask] = n_data_array[i][mask] - bkg
# n_data_array[i][np.logical_and(mask, n_data_array[i] <= 1e-3*bkg)] = 1e-3*bkg
std_bkg[i] = image[np.abs(image-bkg)/bkg < 1.].std()
background[i] = bkg
if subtract_error > 0:
n_data_array, n_error_array, background, error_bkg = subtract_bkg(n_data_array, n_error_array, mask, background, error_bkg)
if display:
display_bkg(data, background, std_bkg, headers, histograms=histograms, binning=binning, coeff=coeff, savename=savename, plots_folder=plots_folder)
return n_data_array, n_error_array, headers, background
return n_data_array, n_error_array, headers, background, error_bkg
def bkg_mini(data, error, mask, headers, sub_shape=(15, 15), subtract_error=True, display=False, savename=None, plots_folder=""):
@@ -440,16 +446,31 @@ def bkg_mini(data, error, mask, headers, sub_shape=(15, 15), subtract_error=True
bkg = np.sqrt(np.sum(sub_image**2)/sub_image.size)*subtract_error if subtract_error > 0 else np.sqrt(np.sum(sub_image**2)/sub_image.size)
error_bkg[i] *= bkg
n_error_array[i] = np.sqrt(n_error_array[i]**2 + error_bkg[i]**2)
# n_error_array[i] = np.sqrt(n_error_array[i]**2 + error_bkg[i]**2)
# Substract background
if subtract_error > 0.:
n_data_array[i][mask] = n_data_array[i][mask] - bkg
n_data_array[i][np.logical_and(mask, n_data_array[i] <= 1e-3*bkg)] = 1e-3*bkg
# if subtract_error > 0.:
# n_data_array[i][mask] = n_data_array[i][mask] - bkg
# n_data_array[i][np.logical_and(mask, n_data_array[i] <= 1e-3*bkg)] = 1e-3*bkg
std_bkg[i] = image[np.abs(image-bkg)/bkg < 1.].std()
background[i] = bkg
if subtract_error > 0:
n_data_array, n_error_array, background, error_bkg = subtract_bkg(n_data_array, n_error_array, mask, background, error_bkg)
if display:
display_bkg(data, background, std_bkg, headers, rectangle=rectangle, savename=savename, plots_folder=plots_folder)
return n_data_array, n_error_array, headers, background
return n_data_array, n_error_array, headers, background, error_bkg
def subtract_bkg(data, error, mask, background, error_bkg):
assert data.ndim == 3, "Input data must have more than 1 image."
n_data_array, n_error_array = deepcopy(data), deepcopy(error)
for i in range(data.shape[0]):
n_data_array[i][mask] = n_data_array[i][mask] - background[i]
n_data_array[i][np.logical_and(mask, n_data_array[i] <= 1e-3 * background[i])] = 1e-3 * background[i]
n_error_array[i] = np.sqrt(n_error_array[i]**2 + error_bkg[i]**2)
return n_data_array, n_error_array, background, error_bkg

2
package/lib/plots.py
View File

@@ -267,7 +267,7 @@ def plot_Stokes(Stokes, savename=None, plots_folder=""):
def polarization_map(Stokes, data_mask=None, rectangle=None, SNRp_cut=3., SNRi_cut=3.,
flux_lim=None, step_vec=1, vec_scale=2., savename=None, plots_folder="", display="default"):
flux_lim=None, step_vec=1, vec_scale=2., savename=None, plots_folder="", display="default", **kwargs):
"""
Plots polarization map from Stokes HDUList.
----------

11
package/lib/reduction.py
View File

@@ -477,17 +477,18 @@ def get_error(data_array, headers, error_array=None, data_mask=None, sub_type=No
err_flat = data*0.03
if (sub_type is None):
n_data_array, c_error_bkg, headers, background = bkg_hist(
n_data_array, c_error_bkg, headers, background, error_bkg = bkg_hist(
data, error, mask, headers, subtract_error=subtract_error, display=display, savename=savename, plots_folder=plots_folder)
elif isinstance(sub_type, str):
if sub_type.lower() in ['auto']:
n_data_array, c_error_bkg, headers, background = bkg_fit(
n_data_array, c_error_bkg, headers, background, error_bkg = bkg_fit(
data, error, mask, headers, subtract_error=subtract_error, display=display, savename=savename, plots_folder=plots_folder)
else:
n_data_array, c_error_bkg, headers, background = bkg_hist(
n_data_array, c_error_bkg, headers, background, error_bkg = bkg_hist(
data, error, mask, headers, sub_type=sub_type, subtract_error=subtract_error, display=display, savename=savename, plots_folder=plots_folder)
elif isinstance(sub_type, tuple):
n_data_array, c_error_bkg, headers, background = bkg_mini(
n_data_array, c_error_bkg, headers, background, error_bkg = bkg_mini(
data, error, mask, headers, sub_shape=sub_type, subtract_error=subtract_error, display=display, savename=savename, plots_folder=plots_folder)
else:
print("Warning: Invalid subtype.")
@@ -496,7 +497,7 @@ def get_error(data_array, headers, error_array=None, data_mask=None, sub_type=No
n_error_array = np.sqrt(err_wav**2+err_psf**2+err_flat**2+c_error_bkg**2)
if return_background:
return n_data_array, n_error_array, headers, background
return n_data_array, n_error_array, headers, background, error_bkg # return background error as well
else:
return n_data_array, n_error_array, headers