From 95bce4523f78e543cb9209cf89d947361534d152 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:20:07 -0400 Subject: [PATCH 01/18] Added logic for len(input_table)==2 --- drizzlepac/haputils/poller_utils.py | 39 ++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 9 deletions(-) mode change 100755 => 100644 drizzlepac/haputils/poller_utils.py diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py old mode 100755 new mode 100644 index d8a74cd7e..c39e9ad31 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -482,7 +482,7 @@ def parse_mvm_tree(det_tree, all_mvm_exposures, log_level): # mvm prod_info = 'skycell_p1234_x01y01 wfc3 uvis f200lp all 2009 1 drz' # prod_list = prod_info.split(" ") - multi_scale = prod_list[2].upper() in ['IR', 'PC'] + multi_scale = prod_list[2].upper() in ['IR'] pscale = 'fine' if not multi_scale else 'coarse' prod_info += " {:s}".format(pscale) @@ -931,10 +931,26 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' poller_dtype = POLLER_DTYPE datasets = [] + + # limit column string types to minimum length formats e.g. str8, str11, etc. obs_converters = {'col4': [ascii.convert_numpy(np.str_)]} + if isinstance(input, str): input_table = ascii.read(input, format='no_header', converters=obs_converters) - if len(input_table.columns) == len(poller_colnames): + + if len(input_table.columns) == 1: + input_table.columns[0].name = 'filename' + is_poller_file = False # gets important keywords from file headers instead of poller file + + # unique logic to collect wfpc2 aperture data from poller file + if len(input_table.columns) == 2: + input_table.columns[0].name = 'filename' + input_table.columns[1].name = 'aperture' + # add dtype for aperture column + poller_dtype+=('str',) + is_poller_file = False + + elif len(input_table.columns) == len(poller_colnames): # We were provided a poller file # Now assign column names to table for i, colname in enumerate(poller_colnames): @@ -1012,10 +1028,12 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' err_msg = "Input image {} missing from current working directory and from the path specified in the poller file. Exiting... ".format(table_line['filename']) log.error(err_msg) raise Exception(err_msg) - elif len(input_table.columns) == 1: - input_table.columns[0].name = 'filename' - is_poller_file = False - + + # input is string with unexpected number of columns + else: + log.error(f'Poller file has an unexpected number of columns, code expects either 1, 2, or {len(poller_colnames)} but received: {len(input_table.columns)}') + raise ValueError + # Since a poller file was the input, it is assumed all the input # data is in the locale directory so just collect the filenames. # datasets = input_table[input_table.colnames[0]].tolist() @@ -1023,6 +1041,7 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' elif isinstance(input, list): filenames = input + input_table= None else: id = '[poller_utils.build_poller_table] ' @@ -1066,7 +1085,10 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' for cname in poller_colnames: cols[cname] = [] cols['filename'] = usable_datasets - + if input_table: + if 'aperture' in input_table.colnames: + cols['aperture'] = input_table['aperture'].tolist() + # If MVM processing and a poller file is the input, this implies there is # only one skycell of interest for all the listed filenames in the poller # file. Establish the WCS, but no need for discovery of overlapping skycells @@ -1120,7 +1142,6 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' poller_names = [colname for colname in cols] poller_table = Table(data=poller_data, names=poller_names, dtype=poller_dtype) - # The input was a poller file, so just keep the viable data rows for output else: good_rows = [] @@ -1184,7 +1205,7 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' sys.exit(0) poller_table = new_poller_table - + return poller_table From 99c41143598c986ce8873a3c819cd582eccee727 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Tue, 17 Oct 2023 18:00:31 -0400 Subject: [PATCH 02/18] added aperture to create_row_info --- drizzlepac/haputils/poller_utils.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index c39e9ad31..1bf590d37 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -146,7 +146,8 @@ def interpret_obset_input(results, log_level): # Now create the output product objects log.debug("Parse the observation set tree and create the exposure, filter, and total detection objects.") obset_dict, tdp_list = parse_obset_tree(obset_tree, log_level) - + import ipdb; ipdb.set_trace() + # This little bit of code adds an attribute to single exposure objects that is True # if a given filter only contains one input (e.g. n_exp = 1) for tot_obj in tdp_list: @@ -198,7 +199,7 @@ def build_obset_tree(obset_table): def create_row_info(row): """Build info string for a row from the obset table""" info_list = [str(row['proposal_id']), "{}".format(row['obset_id']), row['instrument'], - row['detector'], row['filename'][:row['filename'].find('_')], row['filters']] + row['detector'], row['filename'][:row['filename'].find('_')], row['filters'], row['aperture']] return ' '.join(map(str.upper, info_list)), row['filename'] @@ -888,9 +889,12 @@ def determine_filter_name(raw_filter): # ------------------------------------------------------------------------------ -def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm', +def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], poller_type='svm', include_small=True, only_cte=False): - """Create a poller file from dataset names. + """Create a poller file from dataset names for either SMV or MVM processing. Information is either gathered + from the poller file or by using the filename to open the file and pulling information from the header keywords. + The code treats WFPC2 differently, by uses both approaches. For WFPC2, We use simple poller files with a second column + that includes the aperture. The code gathers the rest of the relevant informaiton from the header keywords. Parameters ----------- @@ -942,7 +946,7 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' input_table.columns[0].name = 'filename' is_poller_file = False # gets important keywords from file headers instead of poller file - # unique logic to collect wfpc2 aperture data from poller file + # unique logic to collect WFPC2 aperture data from poller file if len(input_table.columns) == 2: input_table.columns[0].name = 'filename' input_table.columns[1].name = 'aperture' @@ -1039,6 +1043,7 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' # datasets = input_table[input_table.colnames[0]].tolist() filenames = list(input_table.columns[0]) + # If input is a list of filenames elif isinstance(input, list): filenames = input input_table= None From b6abaf6e41fbf99249f53d51b0f2c04f17774272 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Wed, 18 Oct 2023 14:11:16 -0400 Subject: [PATCH 03/18] removed set trace --- drizzlepac/haputils/poller_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index 1bf590d37..587048654 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -146,7 +146,6 @@ def interpret_obset_input(results, log_level): # Now create the output product objects log.debug("Parse the observation set tree and create the exposure, filter, and total detection objects.") obset_dict, tdp_list = parse_obset_tree(obset_tree, log_level) - import ipdb; ipdb.set_trace() # This little bit of code adds an attribute to single exposure objects that is True # if a given filter only contains one input (e.g. n_exp = 1) From 1a2079e78a7d0657e2effdd2492abd2a9a7f852f Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:11:35 -0400 Subject: [PATCH 04/18] elif and else statements added --- drizzlepac/haputils/poller_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index 587048654..f40df7347 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -127,7 +127,6 @@ def interpret_obset_input(results, log_level): """ # set logging level to user-specified level log.setLevel(log_level) - log.debug("Interpret the poller file for the observation set.") obset_table = build_poller_table(results, log_level) # Add INSTRUMENT column @@ -946,7 +945,7 @@ def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], polle is_poller_file = False # gets important keywords from file headers instead of poller file # unique logic to collect WFPC2 aperture data from poller file - if len(input_table.columns) == 2: + elif len(input_table.columns) == 2: input_table.columns[0].name = 'filename' input_table.columns[1].name = 'aperture' # add dtype for aperture column @@ -1092,6 +1091,8 @@ def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], polle if input_table: if 'aperture' in input_table.colnames: cols['aperture'] = input_table['aperture'].tolist() + else: + cols['aperture'] = [' '] * len(usable_datasets) # If MVM processing and a poller file is the input, this implies there is # only one skycell of interest for all the listed filenames in the poller From f008533a6c6ae40d0e1491d3c1cab178deb0dde2 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 19 Oct 2023 09:44:47 -0400 Subject: [PATCH 05/18] catches mvm w/o full poller. Added f strings. --- drizzlepac/haputils/poller_utils.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index f40df7347..e7854e412 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -1019,17 +1019,23 @@ def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], polle # an exception and exit. for table_line in input_table: if os.path.exists(table_line['filename']): - log.info("Input image {} found in current working directory.".format(table_line['filename'])) + log.info(f"Input image {table_line['filename']} found in current working directory.") elif os.path.exists(table_line['pathname']): - log.info("Input image {} not found in current working directory. However, it was found in the path specified in the poller file.".format(table_line['filename'])) + log.info(f"Input image {table_line['filename']} not found in current working directory. However, it was found in the path specified in the poller file.") shutil.copy(table_line['pathname'], os.getcwd()) - log.info("Input image {} copied to current working directory.".format(table_line['pathname'])) + log.info(f"Input image {table_line['pathname']} copied to current working directory.") else: - log.error("Input image {} not found in current working directory.".format(table_line['filename'])) - log.error("Archived input image {} not found.".format(table_line['pathname'])) - err_msg = "Input image {} missing from current working directory and from the path specified in the poller file. Exiting... ".format(table_line['filename']) + log.error(f"Input image {table_line['filename']} not found in current working directory.") + log.error(f"Archived input image {table_line['pathname']} not found.") + err_msg = f"Input image {table_line['filename']} missing from current working directory and from the path specified in the poller file. Exiting... " log.error(err_msg) raise Exception(err_msg) + + elif (poller_type == 'mvm') & (len(input_table.columns) != len(poller_colnames)): + log.error(f"MVMs should use full poller files with {len(poller_colnames)} columns.") + err_msg = f"Full poller files should have {len(poller_colnames)} columns. Exiting... " + log.error(err_msg) + raise Exception(err_msg) # input is string with unexpected number of columns else: From 43c8873b716378ecc2d0e83764f18cd1281c9c6c Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 19 Oct 2023 16:44:43 -0400 Subject: [PATCH 06/18] Passes aperture to objects, breaks code. --- drizzlepac/hapsequencer.py | 7 ++++--- drizzlepac/haputils/poller_utils.py | 14 +++++++------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/drizzlepac/hapsequencer.py b/drizzlepac/hapsequencer.py index 8dcc10ce1..aaad4495b 100755 --- a/drizzlepac/hapsequencer.py +++ b/drizzlepac/hapsequencer.py @@ -765,10 +765,11 @@ def run_align_to_gaia(tot_obj, log_level=logutil.logging.INFO, diagnostic_mode=F for exp_obj in tot_obj.edp_list: if gaia_obj is None: prod_list = exp_obj.info.split("_") - prod_list[4] = "metawcs" + prod_list[5] = "metawcs" gaia_obj = product.FilterProduct(prod_list[0], prod_list[1], prod_list[2], - prod_list[3], prod_list[4], "all", - prod_list[5][0:3], log_level) + prod_list[3], prod_list[4], prod_list[5], "all", + prod_list[6][0:3], log_level) + import ipdb; ipdb.set_trace() gaia_obj.configobj_pars = tot_obj.configobj_pars gaia_obj.add_member(exp_obj) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index e7854e412..189641889 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -197,7 +197,7 @@ def build_obset_tree(obset_table): def create_row_info(row): """Build info string for a row from the obset table""" info_list = [str(row['proposal_id']), "{}".format(row['obset_id']), row['instrument'], - row['detector'], row['filename'][:row['filename'].find('_')], row['filters'], row['aperture']] + row['detector'], row['aperture'], row['filename'][:row['filename'].find('_')], row['filters']] return ' '.join(map(str.upper, info_list)), row['filename'] @@ -675,11 +675,11 @@ def parse_obset_tree(det_tree, log_level): is_grism = True filt_indx -= 1 grism_sep_obj = GrismExposureProduct(prod_list[0], prod_list[1], prod_list[2], - prod_list[3], filename[1], prod_list[5], - prod_list[6], log_level) + prod_list[3], prod_list[4], filename[1], prod_list[6], + prod_list[7], log_level) else: - sep_obj = ExposureProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], - filename[1], prod_list[5], prod_list[6], log_level) + sep_obj = ExposureProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], prod_list[4], + filename[1], prod_list[6], prod_list[7], log_level) # Now that we have defined the ExposureProduct for this input exposure, # do not include it any total or filter product. if not is_member: @@ -699,7 +699,7 @@ def parse_obset_tree(det_tree, log_level): # Create a filter product object for this instrument/detector filt_obj = FilterProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], - prod_list[4], prod_list[5], prod_list[6], log_level) + prod_list[4], prod_list[5], prod_list[6], prod_list[7], log_level) # Append exposure object to the list of exposure objects for this specific filter product object filt_obj.add_member(sep_obj) # Populate filter product dictionary with input filename @@ -712,7 +712,7 @@ def parse_obset_tree(det_tree, log_level): # Create a total detection product object for this instrument/detector tdp_obj = TotalProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], - prod_list[4], prod_list[6], log_level) + prod_list[4], prod_list[5], prod_list[7], log_level) if not is_grism: # Append exposure object to the list of exposure objects for this specific total detection product From 407432898e5021b2b405f3f33b1e39a741e3f8e9 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 19 Oct 2023 16:45:51 -0400 Subject: [PATCH 07/18] added product.py --- drizzlepac/haputils/product.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/drizzlepac/haputils/product.py b/drizzlepac/haputils/product.py index 0c6e94862..ba0e327f0 100755 --- a/drizzlepac/haputils/product.py +++ b/drizzlepac/haputils/product.py @@ -64,7 +64,7 @@ class HAPProduct: """ def __init__( - self, prop_id, obset_id, instrument, detector, filename, filetype, log_level + self, prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level ): # set logging level to user-specified level log.setLevel(log_level) @@ -79,6 +79,7 @@ def __init__( self.obset_id = obset_id self.instrument = instrument self.detector = detector + self.aperture = aperture self.filetype = filetype self.rules_file = None self.basename = ( @@ -536,13 +537,13 @@ class TotalProduct(HAPProduct): """ def __init__( - self, prop_id, obset_id, instrument, detector, filename, filetype, log_level + self, prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, filename, filetype] + [prop_id, obset_id, instrument, detector, aperture, filename, filetype] ) self.exposure_name = filename[0:6] @@ -702,17 +703,18 @@ def __init__( obset_id, instrument, detector, + aperture, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture, filename, filters, filetype] ) if filename[0:7].lower() != "metawcs": self.exposure_name = filename[0:6] @@ -849,17 +851,18 @@ def __init__( obset_id, instrument, detector, + aperture, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture, filename, filters, filetype] ) self.filters = filters self.full_filename = self.copy_exposure(filename) @@ -1014,17 +1017,18 @@ def __init__( obset_id, instrument, detector, + aperture, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture, filename, filters, filetype] ) self.filters = filters self.full_filename = self.copy_exposure(filename) @@ -1110,13 +1114,14 @@ def __init__( obset_id, instrument, detector, + aperture, filename, layer, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level ) filter_str = layer[0] @@ -1345,13 +1350,14 @@ def __init__( obset_id, instrument, detector, + aperture, skycell_name, layer, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, skycell_name, filetype, log_level + prop_id, obset_id, instrument, detector, aperture, skycell_name, filetype, log_level ) # May need to exclude 'filter' component from layer_str filter_str = layer[0] @@ -1369,7 +1375,7 @@ def __init__( layer_scale = layer[1] self.info = "_".join( - ["hst", skycell_name, instrument, detector, filter_str, layer_str] + ["hst", skycell_name, instrument, detector, aperture, filter_str, layer_str] ) self.exposure_name = skycell_name self.cell_id = skycell_name.strip("skycell-") From f3fb64970edc5cc9f5dae4674d88eb520ecf4647 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Wed, 25 Oct 2023 11:23:48 -0400 Subject: [PATCH 08/18] Added aperture col for simple 1-col poller case. --- drizzlepac/hapsequencer.py | 1 - drizzlepac/haputils/poller_utils.py | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/drizzlepac/hapsequencer.py b/drizzlepac/hapsequencer.py index aaad4495b..558f15c04 100755 --- a/drizzlepac/hapsequencer.py +++ b/drizzlepac/hapsequencer.py @@ -769,7 +769,6 @@ def run_align_to_gaia(tot_obj, log_level=logutil.logging.INFO, diagnostic_mode=F gaia_obj = product.FilterProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], prod_list[4], prod_list[5], "all", prod_list[6][0:3], log_level) - import ipdb; ipdb.set_trace() gaia_obj.configobj_pars = tot_obj.configobj_pars gaia_obj.add_member(exp_obj) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index 189641889..1310e2ac9 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -939,9 +939,10 @@ def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], polle if isinstance(input, str): input_table = ascii.read(input, format='no_header', converters=obs_converters) - if len(input_table.columns) == 1: input_table.columns[0].name = 'filename' + input_table['aperture']= 'empty_aperture' + poller_dtype+=('str',) is_poller_file = False # gets important keywords from file headers instead of poller file # unique logic to collect WFPC2 aperture data from poller file @@ -1098,7 +1099,10 @@ def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], polle if 'aperture' in input_table.colnames: cols['aperture'] = input_table['aperture'].tolist() else: - cols['aperture'] = [' '] * len(usable_datasets) + cols['aperture'] = ['aperture'] * len(usable_datasets) + poller_dtype+=('str',) + else: + raise ValueError("Input table is empty. Exiting...") # If MVM processing and a poller file is the input, this implies there is # only one skycell of interest for all the listed filenames in the poller @@ -1160,6 +1164,7 @@ def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], polle for i, old_row in enumerate(input_table): if d == input_table['filename'][i]: good_rows.append(old_row) + import ipdb; ipdb.set_trace() # This table contains the pipeline specified skycell_id for each row # which should be the same value in every row From fc56a376fcf8b2ccdf7f038556e0275736986662 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Wed, 25 Oct 2023 16:07:20 -0400 Subject: [PATCH 09/18] grism issue fix, empty_aperture added in polerfile --- drizzlepac/haputils/poller_utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index 1310e2ac9..b38ddfe35 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -671,7 +671,7 @@ def parse_obset_tree(det_tree, log_level): # Determine if this image is a Grism/Prism or a nominal direct exposure is_grism = False - if prod_list[5].lower().find('g') != -1 or prod_list[5].lower().find('pr') != -1: + if prod_list[6].lower().find('g') != -1 or prod_list[6].lower().find('pr') != -1: is_grism = True filt_indx -= 1 grism_sep_obj = GrismExposureProduct(prod_list[0], prod_list[1], prod_list[2], @@ -704,7 +704,6 @@ def parse_obset_tree(det_tree, log_level): filt_obj.add_member(sep_obj) # Populate filter product dictionary with input filename obset_products[fprod]['files'].append(filename[1]) - # Set up the total detection product dictionary and create a total detection product object # Initialize `info` key for total detection product if not obset_products[totprod]['info']: @@ -713,7 +712,6 @@ def parse_obset_tree(det_tree, log_level): # Create a total detection product object for this instrument/detector tdp_obj = TotalProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], prod_list[4], prod_list[5], prod_list[7], log_level) - if not is_grism: # Append exposure object to the list of exposure objects for this specific total detection product tdp_obj.add_member(sep_obj) @@ -1099,7 +1097,7 @@ def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], polle if 'aperture' in input_table.colnames: cols['aperture'] = input_table['aperture'].tolist() else: - cols['aperture'] = ['aperture'] * len(usable_datasets) + cols['aperture'] = ['empty_aperture'] * len(usable_datasets) poller_dtype+=('str',) else: raise ValueError("Input table is empty. Exiting...") @@ -1164,7 +1162,6 @@ def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], polle for i, old_row in enumerate(input_table): if d == input_table['filename'][i]: good_rows.append(old_row) - import ipdb; ipdb.set_trace() # This table contains the pipeline specified skycell_id for each row # which should be the same value in every row From 7b51cc39dbe59a4b7376422c47a930c3a519382b Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Wed, 25 Oct 2023 16:51:48 -0400 Subject: [PATCH 10/18] line that updates header keyword --- drizzlepac/haputils/processing_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drizzlepac/haputils/processing_utils.py b/drizzlepac/haputils/processing_utils.py index c2b1f158f..eeecbfce1 100644 --- a/drizzlepac/haputils/processing_utils.py +++ b/drizzlepac/haputils/processing_utils.py @@ -169,6 +169,11 @@ def refine_product_headers(product, total_obj_list): # Re-format ACS filter specification if phdu['instrume'] == 'ACS': phdu['filter'] = get_acs_filters(hdu, delimiter=';') + + # WFPC2 update aperture if in poller file + if total_obj_list[0].aperture != 'empty_aperture': + log.info(f"Updating aperture header keyword from {phdu['aperture']} to {total_obj_list[0].aperture}") + phdu['aperture'] = total_obj_list[0].aperture # Insure PHOT* keywords are always in SCI extension for pkw in PHOT_KEYWORDS: From cb0e3bb175aacd44177bbbe84b2c2b576317a920 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 26 Oct 2023 10:45:52 -0400 Subject: [PATCH 11/18] header update only for WFPC2 --- drizzlepac/haputils/processing_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drizzlepac/haputils/processing_utils.py b/drizzlepac/haputils/processing_utils.py index eeecbfce1..1b7cf6ecf 100644 --- a/drizzlepac/haputils/processing_utils.py +++ b/drizzlepac/haputils/processing_utils.py @@ -171,10 +171,12 @@ def refine_product_headers(product, total_obj_list): phdu['filter'] = get_acs_filters(hdu, delimiter=';') # WFPC2 update aperture if in poller file - if total_obj_list[0].aperture != 'empty_aperture': + if (total_obj_list[0].aperture != 'empty_aperture') & (phdu['instrume'] =='WFPC2'): log.info(f"Updating aperture header keyword from {phdu['aperture']} to {total_obj_list[0].aperture}") phdu['aperture'] = total_obj_list[0].aperture - + else: + log.debug("Not updating aperture keyword.") + # Insure PHOT* keywords are always in SCI extension for pkw in PHOT_KEYWORDS: if pkw in phdu: From e7d25b16b0b60f6a87f564844950b2fc644f2435 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 26 Oct 2023 10:51:29 -0400 Subject: [PATCH 12/18] removed multiple type hints --- drizzlepac/haputils/poller_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index b38ddfe35..a2a70f430 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -64,7 +64,7 @@ # ----------------------------------------------------------------------------- # Single Visit Processing Functions # -def interpret_obset_input(results, log_level): +def interpret_obset_input(results: str, log_level): """ Parameters @@ -885,7 +885,7 @@ def determine_filter_name(raw_filter): # ------------------------------------------------------------------------------ -def build_poller_table(input: str | list, log_level, all_mvm_exposures=[], poller_type='svm', +def build_poller_table(input: str, log_level, all_mvm_exposures=[], poller_type='svm', include_small=True, only_cte=False): """Create a poller file from dataset names for either SMV or MVM processing. Information is either gathered from the poller file or by using the filename to open the file and pulling information from the header keywords. From 1641c9ba833ceb696ebbd938b32c3d3818f5b923 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:25:27 -0400 Subject: [PATCH 13/18] proper handling of aperture for full poller --- drizzlepac/haputils/poller_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index a2a70f430..a0c887cd2 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -1097,7 +1097,8 @@ def build_poller_table(input: str, log_level, all_mvm_exposures=[], poller_type= if 'aperture' in input_table.colnames: cols['aperture'] = input_table['aperture'].tolist() else: - cols['aperture'] = ['empty_aperture'] * len(usable_datasets) + add_col = Column(['empty_aperture'] * len(usable_datasets), name='aperture', dtype='str') + input_table.add_column(add_col, index=7) poller_dtype+=('str',) else: raise ValueError("Input table is empty. Exiting...") From dc1cfcfbc9c38de00ba47f85711a9b0178dc6075 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:49:07 -0400 Subject: [PATCH 14/18] Changed aperture name --- drizzlepac/haputils/processing_utils.py | 6 ++-- drizzlepac/haputils/product.py | 38 ++++++++++++------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/drizzlepac/haputils/processing_utils.py b/drizzlepac/haputils/processing_utils.py index 1b7cf6ecf..2c0f4aad4 100644 --- a/drizzlepac/haputils/processing_utils.py +++ b/drizzlepac/haputils/processing_utils.py @@ -171,9 +171,9 @@ def refine_product_headers(product, total_obj_list): phdu['filter'] = get_acs_filters(hdu, delimiter=';') # WFPC2 update aperture if in poller file - if (total_obj_list[0].aperture != 'empty_aperture') & (phdu['instrume'] =='WFPC2'): - log.info(f"Updating aperture header keyword from {phdu['aperture']} to {total_obj_list[0].aperture}") - phdu['aperture'] = total_obj_list[0].aperture + if (total_obj_list[0].aperture_from_poller != 'empty_aperture') & (phdu['instrume'] =='WFPC2'): + log.info(f"Updating aperture header keyword from {phdu['aperture']} to {total_obj_list[0].aperture_from_poller}") + phdu['aperture'] = total_obj_list[0].aperture_from_poller else: log.debug("Not updating aperture keyword.") diff --git a/drizzlepac/haputils/product.py b/drizzlepac/haputils/product.py index ba0e327f0..3dc4b6ba2 100755 --- a/drizzlepac/haputils/product.py +++ b/drizzlepac/haputils/product.py @@ -64,7 +64,7 @@ class HAPProduct: """ def __init__( - self, prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level + self, prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ): # set logging level to user-specified level log.setLevel(log_level) @@ -79,7 +79,7 @@ def __init__( self.obset_id = obset_id self.instrument = instrument self.detector = detector - self.aperture = aperture + self.aperture_from_poller = aperture_from_poller self.filetype = filetype self.rules_file = None self.basename = ( @@ -537,13 +537,13 @@ class TotalProduct(HAPProduct): """ def __init__( - self, prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level + self, prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ): super().__init__( - prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, aperture, filename, filetype] + [prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype] ) self.exposure_name = filename[0:6] @@ -703,18 +703,18 @@ def __init__( obset_id, instrument, detector, - aperture, + aperture_from_poller, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, aperture, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filters, filetype] ) if filename[0:7].lower() != "metawcs": self.exposure_name = filename[0:6] @@ -851,18 +851,18 @@ def __init__( obset_id, instrument, detector, - aperture, + aperture_from_poller, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, aperture, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filters, filetype] ) self.filters = filters self.full_filename = self.copy_exposure(filename) @@ -1017,18 +1017,18 @@ def __init__( obset_id, instrument, detector, - aperture, + aperture_from_poller, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, aperture, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filters, filetype] ) self.filters = filters self.full_filename = self.copy_exposure(filename) @@ -1114,14 +1114,14 @@ def __init__( obset_id, instrument, detector, - aperture, + aperture_from_poller, filename, layer, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, aperture, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) filter_str = layer[0] @@ -1350,14 +1350,14 @@ def __init__( obset_id, instrument, detector, - aperture, + aperture_from_poller, skycell_name, layer, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, aperture, skycell_name, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, skycell_name, filetype, log_level ) # May need to exclude 'filter' component from layer_str filter_str = layer[0] @@ -1375,7 +1375,7 @@ def __init__( layer_scale = layer[1] self.info = "_".join( - ["hst", skycell_name, instrument, detector, aperture, filter_str, layer_str] + ["hst", skycell_name, instrument, detector, aperture_from_poller, filter_str, layer_str] ) self.exposure_name = skycell_name self.cell_id = skycell_name.strip("skycell-") From 0f46a6b81f6a26a6713806d56791b113e322b9ff Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:04:21 -0400 Subject: [PATCH 15/18] added changelog --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 01cceab07..5d08edb25 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,10 @@ number of the code change for that issue. These PRs can be viewed at: 3.6.2rc0 (unreleased) ===================== +- Added functionality to allow the use of a two-column poller file. This is used + to update the WFPC2 SVM aperture header keywords from the values in the poller + file. + - Removed the version restriction on matplotlib. [#1649] - Forced a preferential order on the final selection of the WCS solution From ab834b24c06a2da0a338cb896021de929803bde4 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 26 Oct 2023 15:16:10 -0400 Subject: [PATCH 16/18] Change not update from master. --- drizzlepac/haputils/poller_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index a0c887cd2..ef8bb2d03 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -481,7 +481,7 @@ def parse_mvm_tree(det_tree, all_mvm_exposures, log_level): # mvm prod_info = 'skycell_p1234_x01y01 wfc3 uvis f200lp all 2009 1 drz' # prod_list = prod_info.split(" ") - multi_scale = prod_list[2].upper() in ['IR'] + multi_scale = prod_list[2].upper() in ['IR', 'PC'] pscale = 'fine' if not multi_scale else 'coarse' prod_info += " {:s}".format(pscale) From 1ad96e4c51e163c5e011fa0db4478dc4e34eb6f5 Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 26 Oct 2023 15:54:26 -0400 Subject: [PATCH 17/18] added docs to poller_utils --- drizzlepac/haputils/poller_utils.py | 32 ++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index ef8bb2d03..1389c8c0b 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -91,39 +91,51 @@ def interpret_obset_input(results: str, log_level): Interpret the database query for a given obset to prepare the returned values for use in generating the names of all the expected output products. - Input will have format of:: - + Input will have formated rows of one of the following three options:: + + 1) Full poller file ib4606c5q_flc.fits,11665,B46,06,1.0,F555W,UVIS,/foo/bar/ib4606c5q_flc.fits + + 2) Simpler poller file (list); other info taken from file header keywords + ib4606c5q_flc.fits + + 3) For updating the WFPC2 SVM aperture keyword using the poller file; it + is important that there are no spaces within the poller aperture keyword(s) + + ib4606c5q_flc.fits, PC1-FIX;F160BN15 - which are filename, proposal_id, program_id, obset_id, exptime, filters, detector, pathname. + Full poller files contain filename, proposal_id, program_id, obset_id, exptime, filters, detector, pathname. The output dict will have format (as needed by further code for creating the product filenames) of:: - obs_info_dict["single exposure product 00": {"info": '11665 06 wfc3 uvis ib4606c5q f555w drc', + obs_info_dict["single exposure product 00": {"info": '11665 06 wfc3 uvis empty_aperture ib4606c5q f555w drc', "files": ['ib4606c5q_flc.fits']} . . . - obs_info_dict["single exposure product 08": {"info": '11665 06 wfc3 ir ib4606clq f110w drz', + obs_info_dict["single exposure product 08": {"info": '11665 06 wfc3 ir empty_aperture ib4606clq f110w drz', "files": ['ib4606clq_flt.fits']} - obs_info_dict["filter product 00": {"info": '11665 06 wfc3 uvis ib4606c5q f555w drc', + obs_info_dict["filter product 00": {"info": '11665 06 wfc3 uvis empty_aperture ib4606c5q f555w drc', "files": ['ib4606c5q_flc.fits', 'ib4606c6q_flc.fits']}, . . . - obs_info_dict["filter product 01": {"info": '11665 06 wfc3 ir ib4606cmq f160w drz', + obs_info_dict["filter product 01": {"info": '11665 06 wfc3 ir empty_aperture ib4606cmq f160w drz', "files": ['ib4606cmq_flt.fits', 'ib4606crq_flt.fits']}, - obs_info_dict["total detection product 00": {"info": '11665 06 wfc3 uvis ib4606c5q f555w drc', + obs_info_dict["total detection product 00": {"info": '11665 06 wfc3 uvis empty_aperture ib4606c5q f555w drc', "files": ['ib4606c5q_flc.fits', 'ib4606c6q_flc.fits']} . . . - obs_info_dict["total detection product 01": {"info": '11665 06 wfc3 ir ib4606cmq f160w drz', + obs_info_dict["total detection product 01": {"info": '11665 06 wfc3 ir empty_aperture ib4606cmq f160w drz', "files": ['ib4606cmq_flt.fits', 'ib4606crq_flt.fits']} + The aperture keyword, which has a default value of 'empty_aperture' is filled in the case of WFPC2 observations, + and the use of the two-column format. + """ # set logging level to user-specified level log.setLevel(log_level) @@ -728,7 +740,7 @@ def parse_obset_tree(det_tree, log_level): if is_ccd and len(filt_obj.edp_list) == 1: for e in filt_obj.edp_list: e.crclean = True - + import ipdb;ipdb.set_trace() elif is_grism: tdp_obj.add_grism_member(grism_sep_obj) From dfc1839617604adea17c8ab53ba815fea91c1c1d Mon Sep 17 00:00:00 2001 From: Steve Goldman <32876747+s-goldman@users.noreply.github.com> Date: Thu, 26 Oct 2023 16:02:15 -0400 Subject: [PATCH 18/18] added docs for prod_list filter in poller_utils --- drizzlepac/haputils/poller_utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py index 1389c8c0b..94974f8b2 100644 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -671,15 +671,18 @@ def parse_obset_tree(det_tree, log_level): sep_indx += 1 # Create a single exposure product object - # - # The prod_list[5] is the filter - use this information to distinguish between + + # The GrismExposureProduct is only an attribute of the TotalProduct. + prod_list = exp_prod_info.split(" ") + + # prod_list is 0: proposal_id, 1:observation_id, 2:instrument, 3:detector, + # 4:aperture_from_poller, 5:filename, 6:filters, 7:filetype + + # The prod_list[6] is the filter - use this information to distinguish between # a direct exposure for drizzling (ExposureProduct) and an exposure # (GrismExposureProduct) which is carried along (Grism/Prism) to make analysis # easier for the user by having the same WCS in both the direct and # Grism/Prism products. - # - # The GrismExposureProduct is only an attribute of the TotalProduct. - prod_list = exp_prod_info.split(" ") # Determine if this image is a Grism/Prism or a nominal direct exposure is_grism = False @@ -740,7 +743,6 @@ def parse_obset_tree(det_tree, log_level): if is_ccd and len(filt_obj.edp_list) == 1: for e in filt_obj.edp_list: e.crclean = True - import ipdb;ipdb.set_trace() elif is_grism: tdp_obj.add_grism_member(grism_sep_obj)