diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..bfb2a9b57 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,18 @@ + +Resolves [HLA-nnnn](https://jira.stsci.edu/browse/HLA-nnnn) + + +Closes # + + +This PR addresses ... + +**Checklist for maintainers** +- [ ] added entry in `CHANGELOG.rst` within the relevant release section +- [ ] updated or added relevant tests +- [ ] updated relevant documentation +- [ ] added relevant milestone +- [ ] added relevant label(s) +- [ ] ran regression tests, post a link to the Jenkins job below. + [How to run regression tests on a PR](https://github.com/spacetelescope/jwst/wiki/Running-Regression-Tests-Against-PR-Branches) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..654bc97d6 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,23 @@ +name: build + +on: + release: + types: [ released ] + pull_request: + workflow_dispatch: + +jobs: + build: + uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish.yml@v1 + with: + upload_to_pypi: ${{ (github.event_name == 'release') && (github.event.action == 'released') }} + targets: | + # Linux wheels + - cp3*-manylinux_x86_64 + # MacOS wheels + - cp3*-macosx_x86_64 + # Until we have arm64 runners, we can't automatically test arm64 wheels +# - cp3*-macosx_arm64 + sdist: true + secrets: + pypi_token: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac809040a..898a5b58e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,6 @@ jobs: with: envs: | - linux: check-style - - linux: check-build test: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 needs: [ crds ] diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml deleted file mode 100644 index 6f7d7211d..000000000 --- a/.github/workflows/publish-to-pypi.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Publish to PyPI - -on: - release: - types: [released] - -jobs: - validate: - name: Validate metadata - runs-on: ubuntu-latest - steps: - - uses: spacetelescope/action-publish_to_pypi/validate@master - - build_wheels: - name: Build wheels on ${{ matrix.os }} - needs: [validate] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - - steps: - - uses: spacetelescope/action-publish_to_pypi/build-wheel@master - env: - CIBW_SKIP: "*-manylinux_i686" - - build_sdist: - name: Build source distribution - needs: [validate] - runs-on: ubuntu-latest - steps: - - uses: spacetelescope/action-publish_to_pypi/build-sdist@master - - upload_pypi_platform_wheels: - needs: [build_wheels, build_sdist] - runs-on: ubuntu-latest - steps: - - uses: spacetelescope/action-publish_to_pypi/publish@master - with: - test: false - user: ${{ secrets.PYPI_USERNAME_STSCI_MAINTAINER }} - password: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER }} # WARNING: Do not hardcode secret values here! If you want to use a different user or password, you can override this secret by creating one with the same name in your Github repository settings. - test_password: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER_TEST }} \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8b8dacf17..acdf81c55 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -25,9 +25,8 @@ conda: environment: doc/.rtd-environment.yml python: - system_packages: false install: - method: pip path: . extra_requirements: - - docs \ No newline at end of file + - docs diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 01cceab07..5d08edb25 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -21,6 +21,10 @@ number of the code change for that issue. These PRs can be viewed at: 3.6.2rc0 (unreleased) ===================== +- Added functionality to allow the use of a two-column poller file. This is used + to update the WFPC2 SVM aperture header keywords from the values in the poller + file. + - Removed the version restriction on matplotlib. [#1649] - Forced a preferential order on the final selection of the WCS solution diff --git a/JenkinsfileRT b/JenkinsfileRT index c38f36f25..a0d8272e5 100644 --- a/JenkinsfileRT +++ b/JenkinsfileRT @@ -22,10 +22,10 @@ bc1.nodetype = 'linux' bc1.env_vars = ['TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory'] bc1.name = '3.9' bc1.conda_packages = ['python=3.9'] -bc1.build_cmds = ["pip install numpy astropy codecov pytest-cov ci-watson==0.5", +bc1.build_cmds = ["pip install numpy astropy codecov pytest-cov ci-watson", "pip install --upgrade -e '.[test]'", "pip freeze"] -bc1.test_cmds = ["pytest --cov=./ --basetemp=tests_output --junitxml results.xml --bigdata", +bc1.test_cmds = ["pytest --cov=./ --basetemp=tests_output --junitxml=results.xml --bigdata", "codecov"] bc1.test_configs = [data_config] bc1.failedFailureThresh = 0 @@ -38,20 +38,18 @@ bc3 = new BuildConfig() bc3.runtime.add('CFLAGS=-std=gnu99') bc3.nodetype = 'linux' bc3.env_vars = ['TEST_BIGDATA=https://bytesalad.stsci.edu/artifactory'] -bc3.name = '3.10-dev' +bc3.name = '3.10' bc3.conda_packages = ['python=3.10'] -bc3.build_cmds = ["pip install numpy astropy codecov pytest-cov ci-watson==0.5", - "pip install -r requirements-dev.txt --upgrade -e '.[test]'", +bc3.build_cmds = ["pip install numpy astropy ci-watson", + "pip install --upgrade -e '.[test]'", "pip freeze"] -bc3.test_cmds = ["pytest --cov=./ --basetemp=tests_output --bigdata", - "codecov"] bc3.test_configs = [data_config] bc4 = utils.copy(bc3) -bc4.name = '3.11-dev' +bc4.name = '3.11' bc4.conda_packages = ['python=3.11'] // Iterate over configurations that define the (distributed) build matrix. // Spawn a host (or workdir) for each combination and run in parallel. // Also apply the job configuration defined in `jobconfig` above. -utils.run([bc1, bc3, bc4, jobconfig]) +utils.run([bc1, bc2, bc3, bc4, jobconfig]) diff --git a/README.md b/README.md index b4eb79d5b..2d5c6b269 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Drizzlepac -[![Build Status](https://dev.azure.com/spacetelescope/drizzlepac/_apis/build/status/spacetelescope.drizzlepac?branchName=master)](https://dev.azure.com/spacetelescope/drizzlepac/_build/latest?definitionId=2&branchName=master) -[![Build Status](https://ssbjenkins.stsci.edu/job/STScI/job/drizzlepac/job/master/badge/icon)](https://ssbjenkins.stsci.edu/job/STScI/job/drizzlepac/job/master/) +[![Build Status](https://github.com/spacetelescope/drizzlepac/actions/workflows/ci.yml/badge.svg)](https://github.com/spacetelescope/drizzlepac/actions/workflows/ci.yml) [![Documentation Status](https://readthedocs.org/projects/drizzlepac/badge/?version=latest)](http://drizzlepac.readthedocs.io/en/latest/?badge=latest) [![codecov](https://codecov.io/gh/spacetelescope/drizzlepac/branch/master/graph/badge.svg)](https://codecov.io/gh/spacetelescope/drizzlepac) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.3743274.svg)](https://doi.org/10.5281/zenodo.3743274) @@ -23,10 +22,7 @@ can be found at: ## Conda (Recommended) -```bash -$ conda config --add channels http://ssb.stsci.edu/astroconda -$ conda create -n astroconda stsci -``` +`Drizzlepac` is installed when you install the `stenv` conda environment (a replacement for `astroconda`). Select your desired release and follow the instructions on the [`stenv` installation page](https://stenv.readthedocs.io/en/latest/getting_started.html). ## From Source diff --git a/doc/source/getting_started/installation.rst b/doc/source/getting_started/installation.rst index 4b510df10..22c9b9d68 100644 --- a/doc/source/getting_started/installation.rst +++ b/doc/source/getting_started/installation.rst @@ -1,4 +1,47 @@ Installation ------------ -*Coming soon!* \ No newline at end of file +Conda (Recommended) +=================== + +``Drizzlepac`` is installed when you install the ``stenv`` conda environment (a replacement for ``astroconda``). Select your desired release and follow the instructions on the `installation page `_. + + +From Source +=========== + +Clone this repository +********************* +.. code-block:: shell + + git clone https://github.com/spacetelescope/drizzlepac + cd drizzlepac + + +Build the documentation +======================= + +*Note:* If you intend to use ``drizzlepac``'s embedded help feature from within +an interactive ``python`` or ``ipython`` session, we recommend you do not skip +this step. + + +.. code-block:: shell + + cd doc/ + make html + + +Install drizzlepac +================== + +.. code-block:: shell + + pip install . + +The option ``--no-use-pep517`` MAY be required in order to correctly build +the C extensions with ``pip`` versions up to 22.2, after commenting out +the ``build-backend`` from the ``pyproject.toml`` config file. + +**Support for installing using `pip` is still evolving, so use of this +command is provided on an experimental basis for now.** diff --git a/drizzlepac/align.py b/drizzlepac/align.py index 5269c2742..f6e6dadbe 100644 --- a/drizzlepac/align.py +++ b/drizzlepac/align.py @@ -367,7 +367,7 @@ def perform_align( hdr0 = fits.getheader(imglist[0]) inst = hdr0.get("instrume") if inst.lower() == "wfpc2" and "detector" not in hdr0: - det = "wfpc2" + det = "pc" else: det = hdr0.get("detector") apars = get_default_pars(inst, det) @@ -961,7 +961,7 @@ def determine_fit_quality( if catalogs_remaining: log.warning( "Not enough cross matches found between astrometric" - " catalog and sources found in {}".format(image_name) + " catalog and sources found in {} ()".format(image_name, num_xmatches) ) continue @@ -1158,7 +1158,7 @@ def determine_fit_quality( if not overall_valid: log.info("The fit solution for some or all of the images is not valid.") if max_rms_val > auto_good_rms or not overall_valid: - log.info("Try again with the next catalog") + log.info("Trying again with the next catalog, method, or geometry depending upon the current fitting cycle.") else: log.info("Fit calculations successful.") diff --git a/drizzlepac/hapsequencer.py b/drizzlepac/hapsequencer.py index 0a9fb20b9..558f15c04 100755 --- a/drizzlepac/hapsequencer.py +++ b/drizzlepac/hapsequencer.py @@ -94,7 +94,7 @@ "SVM_CATALOG_WFC": 'on', "SVM_CATALOG_UVIS": 'on', "SVM_CATALOG_IR": 'on', - "SVM_CATALOG_WFPC2": 'on'} + "SVM_CATALOG_PC": 'on'} envvar_cat_str = "SVM_CATALOG_{}" # -------------------------------------------------------------------------------------------------------------- @@ -122,7 +122,7 @@ def create_catalog_products(total_obj_list, log_level, diagnostic_mode=False, ph Specify which, if any, catalogs should be generated at all, based on detector. This dictionary needs to contain values for all instruments; namely: - SVM_CATALOG_HRC, SVM_CATALOG_SBC, SVM_CATALOG_WFC, SVM_CATALOG_UVIS, SVM_CATALOG_IR, SVM_CATALOG_WFPC2 + SVM_CATALOG_HRC, SVM_CATALOG_SBC, SVM_CATALOG_WFC, SVM_CATALOG_UVIS, SVM_CATALOG_IR, SVM_CATALOG_PC These variables can be defined with values of 'on'/'off'/'yes'/'no'/'true'/'false'. @@ -765,10 +765,10 @@ def run_align_to_gaia(tot_obj, log_level=logutil.logging.INFO, diagnostic_mode=F for exp_obj in tot_obj.edp_list: if gaia_obj is None: prod_list = exp_obj.info.split("_") - prod_list[4] = "metawcs" + prod_list[5] = "metawcs" gaia_obj = product.FilterProduct(prod_list[0], prod_list[1], prod_list[2], - prod_list[3], prod_list[4], "all", - prod_list[5][0:3], log_level) + prod_list[3], prod_list[4], prod_list[5], "all", + prod_list[6][0:3], log_level) gaia_obj.configobj_pars = tot_obj.configobj_pars gaia_obj.add_member(exp_obj) diff --git a/drizzlepac/haputils/align_utils.py b/drizzlepac/haputils/align_utils.py index 750f5a895..8a13ef227 100755 --- a/drizzlepac/haputils/align_utils.py +++ b/drizzlepac/haputils/align_utils.py @@ -174,7 +174,7 @@ def __init__(self, input_list, clobber=False, dqname='DQ', process_type='', hdr0 = fits.getheader(img) instrume = hdr0.get('instrume') if instrume.lower() == 'wfpc2' and 'detector' not in hdr0: - detector = 'WFPC2' + detector = 'PC' else: detector = hdr0.get('detector') diff --git a/drizzlepac/haputils/astrometric_utils.py b/drizzlepac/haputils/astrometric_utils.py index 5b0dc31ec..0c6777fdb 100644 --- a/drizzlepac/haputils/astrometric_utils.py +++ b/drizzlepac/haputils/astrometric_utils.py @@ -167,6 +167,7 @@ 'mag': 'mag', 'objID': 'objID', 'epoch': 'epoch'}, } +log.info(f'ASTROMETRIC_CATALOG_URL = {SERVICELOCATION}') # CRBIT definitions CRBIT = 4096 @@ -181,7 +182,7 @@ def create_astrometric_catalog(inputs, catalog="GAIAedr3", output="ref_cat.ecsv" gaia_only=False, table_format="ascii.ecsv", existing_wcs=None, num_sources=None, use_footprint=False, full_catalog=False, - user_epoch='match', log_level=logutil.logging.NOTSET): + user_epoch='match', log_level=logutil.logging.INFO): """Create an astrometric catalog that covers the inputs' field-of-view. This function will return a table containing sources derived from the @@ -451,7 +452,7 @@ def get_catalog(ra, dec, sr=0.1, epoch=None, catalog='GSC241'): spec = base_spec + epoch_str.format(epoch) if epoch else base_spec serviceUrl = '{}/{}?{}'.format(SERVICELOCATION, serviceType, spec) - log.debug("Getting catalog using: \n {}".format(serviceUrl)) + log.info(f"Getting catalog using: \n {serviceUrl}") rawcat = requests.get(serviceUrl, headers=headers) r_contents = rawcat.content.decode() # convert from bytes to a String rstr = r_contents.split('\r\n') @@ -460,7 +461,7 @@ def get_catalog(ra, dec, sr=0.1, epoch=None, catalog='GSC241'): if rstr[0].startswith('Error'): # Try again without EPOCH serviceUrl = '{}/{}?{}'.format(SERVICELOCATION, serviceType, base_spec) - log.debug("Getting catalog using: \n {}".format(serviceUrl)) + log.warning(f"Problem accessing catalog service - getting catalog using: \n {serviceUrl}") rawcat = requests.get(serviceUrl, headers=headers) r_contents = rawcat.content.decode() # convert from bytes to a String rstr = r_contents.split('\r\n') @@ -522,7 +523,7 @@ def get_catalog_from_footprint(footprint, epoch=None, catalog='GSC241'): spec = base_spec + epoch_str.format(epoch) if epoch else base_spec serviceUrl = '{}/{}?{}'.format(SERVICELOCATION, serviceType, spec) - log.debug("Getting catalog using: \n {}".format(serviceUrl)) + log.info(f"Getting catalog from footprint using: \n {serviceUrl}") rawcat = requests.get(serviceUrl, headers=headers) r_contents = rawcat.content.decode() # convert from bytes to a String @@ -532,7 +533,7 @@ def get_catalog_from_footprint(footprint, epoch=None, catalog='GSC241'): if rstr[0].startswith('Error'): # Try again without EPOCH serviceUrl = '{}/{}?{}'.format(SERVICELOCATION, serviceType, base_spec) - log.debug("Getting catalog using: \n {}".format(serviceUrl)) + log.warning(f"Problem accessing catalog service - getting catalog from footprint using: \n {serviceUrl}") rawcat = requests.get(serviceUrl, headers=headers) r_contents = rawcat.content.decode() # convert from bytes to a String rstr = r_contents.split('\r\n') @@ -719,7 +720,7 @@ def compute_2d_background(imgarr, box_size, win_size, def build_auto_kernel(imgarr, whtarr, fwhm=3.0, threshold=None, source_box=7, good_fwhm=[1.0, 4.0], num_fwhm=30, - isolation_size=11, saturation_limit=70000., log_level=logutil.logging.NOTSET): + isolation_size=11, saturation_limit=70000., log_level=logutil.logging.INFO): """Build kernel for use in source detection based on image PSF This algorithm looks for an isolated point-source that is non-saturated to use as a template for the source detection kernel. Failing to find any suitable sources, it will return a @@ -898,7 +899,7 @@ def find_fwhm(psf, default_fwhm): def extract_point_sources(img, dqmask=None, fwhm=3.0, kernel=None, nbright=1000, - threshold=200.0, sigma=3.0, source_box=7, log_level=logutil.logging.NOTSET): + threshold=200.0, sigma=3.0, source_box=7, log_level=logutil.logging.INFO): """Use photutils to replicate the IRAF point-source catalogs""" # Initialize logging for this user-callable function log.setLevel(log_level) @@ -961,7 +962,7 @@ def extract_sources(img, dqmask=None, fwhm=3.0, kernel=None, photmode=None, dao_nsigma=3.0, source_box=7, classify=True, centering_mode="starfind", nlargest=None, outroot=None, plot=False, vmax=None, deblend=False, - log_level=logutil.logging.NOTSET): + log_level=logutil.logging.INFO): """Use photutils to find sources in image based on segmentation. Parameters diff --git a/drizzlepac/haputils/catalog_utils.py b/drizzlepac/haputils/catalog_utils.py index 644759e32..d19d4a7d9 100755 --- a/drizzlepac/haputils/catalog_utils.py +++ b/drizzlepac/haputils/catalog_utils.py @@ -813,11 +813,14 @@ def __init__(self, image, param_dict, param_dict_qc, diagnostic_mode, tp_sources scale = max(self.param_dict['scale'], self_scale) self.tp_masks = None if not self.image.blank: - self.tp_masks = make_wht_masks(self.image.wht_image, self.image.inv_footprint_mask, - scale=scale, - sensitivity=self.param_dict['sensitivity'], - kernel=(self.param_dict['region_size'], - self.param_dict['region_size'])) + self.tp_masks = make_wht_masks( + self.image.wht_image, + self.image.inv_footprint_mask, + scale=scale, + sensitivity=self.param_dict['sensitivity'], + kernel=(self.param_dict['region_size'], + self.param_dict['region_size']) + ) def identify_sources(self, **pars): pass @@ -2905,28 +2908,50 @@ def make_inv_mask(mask): return invmask -def make_wht_masks(whtarr, maskarr, scale=1.5, sensitivity=0.95, kernel=(11, 11)): - +def make_wht_masks( + whtarr: np.ndarray, # image weight array (dtype=float32), zeros outside of footprint + maskarr: np.ndarray, # mask array (dtype=bool), typically inverse of footprint + scale: float = 1.5, + sensitivity: float = 0.95, + kernel: tuple = (11, 11) # kernel size for maximum filter window +) -> list: # list containing a dictionary + """ Uses scipy's maximum_filter to create the image weight masks of floats between 0 and 1. + Function produces a list including a dictionary with the scale (float), wht_limit (float), + mask (np.ndarray of bools, dtype=int16), and relative weight (np.ndarray, dtype=float32). + + """ + + # create inverse of mask as ints invmask = make_inv_mask(maskarr) + # uses scipy maximum filter on image. Maximum filter selects the largest value within an ordered + # window of pixels values and replaces the central pixel with the largest value. maxwht = ndimage.filters.maximum_filter(whtarr, size=kernel) + + # normalized weight array rel_wht = maxwht / maxwht.max() + # initialize values delta = 0.0 master_mask = np.zeros(invmask.shape, dtype=np.uint16) limit = 1 / scale masks = [] + + # loop through scale values until delta is greater than sensitivity while delta < sensitivity: - mask = rel_wht > limit mask = (mask.astype(np.uint16) * invmask) - master_mask new_delta = master_mask.sum() / mask.sum() if new_delta < sensitivity: - masks.append(dict(scale=limit, - wht_limit=limit * maxwht.max(), - mask=mask, - rel_weight=rel_wht * mask)) + masks.append( + dict( + scale=limit, + wht_limit=limit * maxwht.max(), + mask=mask, + rel_weight=rel_wht * mask, + ) + ) delta = new_delta master_mask = master_mask + mask diff --git a/drizzlepac/haputils/cell_utils.py b/drizzlepac/haputils/cell_utils.py index 88dfaa1cf..797737496 100644 --- a/drizzlepac/haputils/cell_utils.py +++ b/drizzlepac/haputils/cell_utils.py @@ -1662,8 +1662,8 @@ def fill_row_k(imin, imax, k, S, J, sum_x, sum_x_sq, N): fill_row_k(i + 1, imax, k, S, J, sum_x, sum_x_sq, N) def fill_dp_matrix(data, S, J, K, N): - sum_x = np.zeros(N, dtype=np.float_) - sum_x_sq = np.zeros(N, dtype=np.float_) + sum_x = np.zeros(N, dtype=np.float64) + sum_x_sq = np.zeros(N, dtype=np.float64) # median. used to shift the values of x to improve numerical stability shift = data[N // 2] @@ -1702,7 +1702,7 @@ def ckmeans(data, n_clusters): data.sort() n = len(data) - S = np.zeros((n_clusters, n), dtype=np.float_) + S = np.zeros((n_clusters, n), dtype=np.float64) J = np.zeros((n_clusters, n), dtype=np.uint64) diff --git a/drizzlepac/haputils/config_utils.py b/drizzlepac/haputils/config_utils.py index 7144fb994..441e65fd1 100644 --- a/drizzlepac/haputils/config_utils.py +++ b/drizzlepac/haputils/config_utils.py @@ -248,14 +248,14 @@ def _determine_conditions(self, prod_obj): elif self.instrument == "wfpc2": if self.hap_pipeline_name == 'mvm': if n_exp > 1: - self.conditions.append("wfpc2_wfpc2_any_n2") + self.conditions.append("wfpc2_pc_any_n2") if self.hap_pipeline_name == 'svm': if n_exp == 2: - self.conditions.append("wfpc2_wfpc2_any_n2") + self.conditions.append("wfpc2_pc_any_n2") if n_exp == 3: - self.conditions.append("wfpc2_wfpc2_any_n3") + self.conditions.append("wfpc2_pc_any_n3") if n_exp >= 4: - self.conditions.append("wfpc2_wfpc2_any_n4") + self.conditions.append("wfpc2_pc_any_n4") else: log.error("{} is an invalid HST instrument!".format(self.instrument)) sys.exit(1) @@ -672,7 +672,7 @@ def read_index(instrument, detector, hap_pipeline_name='svm'): def get_wfpc2_pars(infiles): pars = {} - full_cfg_index, pars_dir = read_index('wfpc2', 'wfpc2', hap_pipeline_name='svm') + full_cfg_index, pars_dir = read_index('wfpc2', 'pc', hap_pipeline_name='svm') hap_pipeline_name = 'svm' conditions = ["single_basic"] diff --git a/drizzlepac/haputils/poller_utils.py b/drizzlepac/haputils/poller_utils.py old mode 100755 new mode 100644 index a9b0f43ce..94974f8b2 --- a/drizzlepac/haputils/poller_utils.py +++ b/drizzlepac/haputils/poller_utils.py @@ -64,7 +64,7 @@ # ----------------------------------------------------------------------------- # Single Visit Processing Functions # -def interpret_obset_input(results, log_level): +def interpret_obset_input(results: str, log_level): """ Parameters @@ -91,43 +91,54 @@ def interpret_obset_input(results, log_level): Interpret the database query for a given obset to prepare the returned values for use in generating the names of all the expected output products. - Input will have format of:: - + Input will have formated rows of one of the following three options:: + + 1) Full poller file ib4606c5q_flc.fits,11665,B46,06,1.0,F555W,UVIS,/foo/bar/ib4606c5q_flc.fits + + 2) Simpler poller file (list); other info taken from file header keywords + ib4606c5q_flc.fits + + 3) For updating the WFPC2 SVM aperture keyword using the poller file; it + is important that there are no spaces within the poller aperture keyword(s) + + ib4606c5q_flc.fits, PC1-FIX;F160BN15 - which are filename, proposal_id, program_id, obset_id, exptime, filters, detector, pathname. + Full poller files contain filename, proposal_id, program_id, obset_id, exptime, filters, detector, pathname. The output dict will have format (as needed by further code for creating the product filenames) of:: - obs_info_dict["single exposure product 00": {"info": '11665 06 wfc3 uvis ib4606c5q f555w drc', + obs_info_dict["single exposure product 00": {"info": '11665 06 wfc3 uvis empty_aperture ib4606c5q f555w drc', "files": ['ib4606c5q_flc.fits']} . . . - obs_info_dict["single exposure product 08": {"info": '11665 06 wfc3 ir ib4606clq f110w drz', + obs_info_dict["single exposure product 08": {"info": '11665 06 wfc3 ir empty_aperture ib4606clq f110w drz', "files": ['ib4606clq_flt.fits']} - obs_info_dict["filter product 00": {"info": '11665 06 wfc3 uvis ib4606c5q f555w drc', + obs_info_dict["filter product 00": {"info": '11665 06 wfc3 uvis empty_aperture ib4606c5q f555w drc', "files": ['ib4606c5q_flc.fits', 'ib4606c6q_flc.fits']}, . . . - obs_info_dict["filter product 01": {"info": '11665 06 wfc3 ir ib4606cmq f160w drz', + obs_info_dict["filter product 01": {"info": '11665 06 wfc3 ir empty_aperture ib4606cmq f160w drz', "files": ['ib4606cmq_flt.fits', 'ib4606crq_flt.fits']}, - obs_info_dict["total detection product 00": {"info": '11665 06 wfc3 uvis ib4606c5q f555w drc', + obs_info_dict["total detection product 00": {"info": '11665 06 wfc3 uvis empty_aperture ib4606c5q f555w drc', "files": ['ib4606c5q_flc.fits', 'ib4606c6q_flc.fits']} . . . - obs_info_dict["total detection product 01": {"info": '11665 06 wfc3 ir ib4606cmq f160w drz', + obs_info_dict["total detection product 01": {"info": '11665 06 wfc3 ir empty_aperture ib4606cmq f160w drz', "files": ['ib4606cmq_flt.fits', 'ib4606crq_flt.fits']} + The aperture keyword, which has a default value of 'empty_aperture' is filled in the case of WFPC2 observations, + and the use of the two-column format. + """ # set logging level to user-specified level log.setLevel(log_level) - log.debug("Interpret the poller file for the observation set.") obset_table = build_poller_table(results, log_level) # Add INSTRUMENT column @@ -146,7 +157,7 @@ def interpret_obset_input(results, log_level): # Now create the output product objects log.debug("Parse the observation set tree and create the exposure, filter, and total detection objects.") obset_dict, tdp_list = parse_obset_tree(obset_tree, log_level) - + # This little bit of code adds an attribute to single exposure objects that is True # if a given filter only contains one input (e.g. n_exp = 1) for tot_obj in tdp_list: @@ -198,7 +209,7 @@ def build_obset_tree(obset_table): def create_row_info(row): """Build info string for a row from the obset table""" info_list = [str(row['proposal_id']), "{}".format(row['obset_id']), row['instrument'], - row['detector'], row['filename'][:row['filename'].find('_')], row['filters']] + row['detector'], row['aperture'], row['filename'][:row['filename'].find('_')], row['filters']] return ' '.join(map(str.upper, info_list)), row['filename'] @@ -482,7 +493,7 @@ def parse_mvm_tree(det_tree, all_mvm_exposures, log_level): # mvm prod_info = 'skycell_p1234_x01y01 wfc3 uvis f200lp all 2009 1 drz' # prod_list = prod_info.split(" ") - multi_scale = prod_list[2].upper() in ['IR'] + multi_scale = prod_list[2].upper() in ['IR', 'PC'] pscale = 'fine' if not multi_scale else 'coarse' prod_info += " {:s}".format(pscale) @@ -660,27 +671,30 @@ def parse_obset_tree(det_tree, log_level): sep_indx += 1 # Create a single exposure product object - # - # The prod_list[5] is the filter - use this information to distinguish between + + # The GrismExposureProduct is only an attribute of the TotalProduct. + prod_list = exp_prod_info.split(" ") + + # prod_list is 0: proposal_id, 1:observation_id, 2:instrument, 3:detector, + # 4:aperture_from_poller, 5:filename, 6:filters, 7:filetype + + # The prod_list[6] is the filter - use this information to distinguish between # a direct exposure for drizzling (ExposureProduct) and an exposure # (GrismExposureProduct) which is carried along (Grism/Prism) to make analysis # easier for the user by having the same WCS in both the direct and # Grism/Prism products. - # - # The GrismExposureProduct is only an attribute of the TotalProduct. - prod_list = exp_prod_info.split(" ") # Determine if this image is a Grism/Prism or a nominal direct exposure is_grism = False - if prod_list[5].lower().find('g') != -1 or prod_list[5].lower().find('pr') != -1: + if prod_list[6].lower().find('g') != -1 or prod_list[6].lower().find('pr') != -1: is_grism = True filt_indx -= 1 grism_sep_obj = GrismExposureProduct(prod_list[0], prod_list[1], prod_list[2], - prod_list[3], filename[1], prod_list[5], - prod_list[6], log_level) + prod_list[3], prod_list[4], filename[1], prod_list[6], + prod_list[7], log_level) else: - sep_obj = ExposureProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], - filename[1], prod_list[5], prod_list[6], log_level) + sep_obj = ExposureProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], prod_list[4], + filename[1], prod_list[6], prod_list[7], log_level) # Now that we have defined the ExposureProduct for this input exposure, # do not include it any total or filter product. if not is_member: @@ -700,12 +714,11 @@ def parse_obset_tree(det_tree, log_level): # Create a filter product object for this instrument/detector filt_obj = FilterProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], - prod_list[4], prod_list[5], prod_list[6], log_level) + prod_list[4], prod_list[5], prod_list[6], prod_list[7], log_level) # Append exposure object to the list of exposure objects for this specific filter product object filt_obj.add_member(sep_obj) # Populate filter product dictionary with input filename obset_products[fprod]['files'].append(filename[1]) - # Set up the total detection product dictionary and create a total detection product object # Initialize `info` key for total detection product if not obset_products[totprod]['info']: @@ -713,8 +726,7 @@ def parse_obset_tree(det_tree, log_level): # Create a total detection product object for this instrument/detector tdp_obj = TotalProduct(prod_list[0], prod_list[1], prod_list[2], prod_list[3], - prod_list[4], prod_list[6], log_level) - + prod_list[4], prod_list[5], prod_list[7], log_level) if not is_grism: # Append exposure object to the list of exposure objects for this specific total detection product tdp_obj.add_member(sep_obj) @@ -731,7 +743,6 @@ def parse_obset_tree(det_tree, log_level): if is_ccd and len(filt_obj.edp_list) == 1: for e in filt_obj.edp_list: e.crclean = True - elif is_grism: tdp_obj.add_grism_member(grism_sep_obj) @@ -888,9 +899,12 @@ def determine_filter_name(raw_filter): # ------------------------------------------------------------------------------ -def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm', +def build_poller_table(input: str, log_level, all_mvm_exposures=[], poller_type='svm', include_small=True, only_cte=False): - """Create a poller file from dataset names. + """Create a poller file from dataset names for either SMV or MVM processing. Information is either gathered + from the poller file or by using the filename to open the file and pulling information from the header keywords. + The code treats WFPC2 differently, by uses both approaches. For WFPC2, We use simple poller files with a second column + that includes the aperture. The code gathers the rest of the relevant informaiton from the header keywords. Parameters ----------- @@ -931,10 +945,27 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' poller_dtype = POLLER_DTYPE datasets = [] + + # limit column string types to minimum length formats e.g. str8, str11, etc. obs_converters = {'col4': [ascii.convert_numpy(np.str_)]} + if isinstance(input, str): input_table = ascii.read(input, format='no_header', converters=obs_converters) - if len(input_table.columns) == len(poller_colnames): + if len(input_table.columns) == 1: + input_table.columns[0].name = 'filename' + input_table['aperture']= 'empty_aperture' + poller_dtype+=('str',) + is_poller_file = False # gets important keywords from file headers instead of poller file + + # unique logic to collect WFPC2 aperture data from poller file + elif len(input_table.columns) == 2: + input_table.columns[0].name = 'filename' + input_table.columns[1].name = 'aperture' + # add dtype for aperture column + poller_dtype+=('str',) + is_poller_file = False + + elif len(input_table.columns) == len(poller_colnames): # We were provided a poller file # Now assign column names to table for i, colname in enumerate(poller_colnames): @@ -1001,28 +1032,38 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' # an exception and exit. for table_line in input_table: if os.path.exists(table_line['filename']): - log.info("Input image {} found in current working directory.".format(table_line['filename'])) + log.info(f"Input image {table_line['filename']} found in current working directory.") elif os.path.exists(table_line['pathname']): - log.info("Input image {} not found in current working directory. However, it was found in the path specified in the poller file.".format(table_line['filename'])) + log.info(f"Input image {table_line['filename']} not found in current working directory. However, it was found in the path specified in the poller file.") shutil.copy(table_line['pathname'], os.getcwd()) - log.info("Input image {} copied to current working directory.".format(table_line['pathname'])) + log.info(f"Input image {table_line['pathname']} copied to current working directory.") else: - log.error("Input image {} not found in current working directory.".format(table_line['filename'])) - log.error("Archived input image {} not found.".format(table_line['pathname'])) - err_msg = "Input image {} missing from current working directory and from the path specified in the poller file. Exiting... ".format(table_line['filename']) + log.error(f"Input image {table_line['filename']} not found in current working directory.") + log.error(f"Archived input image {table_line['pathname']} not found.") + err_msg = f"Input image {table_line['filename']} missing from current working directory and from the path specified in the poller file. Exiting... " log.error(err_msg) raise Exception(err_msg) - elif len(input_table.columns) == 1: - input_table.columns[0].name = 'filename' - is_poller_file = False - + + elif (poller_type == 'mvm') & (len(input_table.columns) != len(poller_colnames)): + log.error(f"MVMs should use full poller files with {len(poller_colnames)} columns.") + err_msg = f"Full poller files should have {len(poller_colnames)} columns. Exiting... " + log.error(err_msg) + raise Exception(err_msg) + + # input is string with unexpected number of columns + else: + log.error(f'Poller file has an unexpected number of columns, code expects either 1, 2, or {len(poller_colnames)} but received: {len(input_table.columns)}') + raise ValueError + # Since a poller file was the input, it is assumed all the input # data is in the locale directory so just collect the filenames. # datasets = input_table[input_table.colnames[0]].tolist() filenames = list(input_table.columns[0]) + # If input is a list of filenames elif isinstance(input, list): filenames = input + input_table= None else: id = '[poller_utils.build_poller_table] ' @@ -1066,7 +1107,16 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' for cname in poller_colnames: cols[cname] = [] cols['filename'] = usable_datasets - + if input_table: + if 'aperture' in input_table.colnames: + cols['aperture'] = input_table['aperture'].tolist() + else: + add_col = Column(['empty_aperture'] * len(usable_datasets), name='aperture', dtype='str') + input_table.add_column(add_col, index=7) + poller_dtype+=('str',) + else: + raise ValueError("Input table is empty. Exiting...") + # If MVM processing and a poller file is the input, this implies there is # only one skycell of interest for all the listed filenames in the poller # file. Establish the WCS, but no need for discovery of overlapping skycells @@ -1120,7 +1170,6 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' poller_names = [colname for colname in cols] poller_table = Table(data=poller_data, names=poller_names, dtype=poller_dtype) - # The input was a poller file, so just keep the viable data rows for output else: good_rows = [] @@ -1184,7 +1233,7 @@ def build_poller_table(input, log_level, all_mvm_exposures=[], poller_type='svm' sys.exit(0) poller_table = new_poller_table - + return poller_table diff --git a/drizzlepac/haputils/processing_utils.py b/drizzlepac/haputils/processing_utils.py index c2b1f158f..2c0f4aad4 100644 --- a/drizzlepac/haputils/processing_utils.py +++ b/drizzlepac/haputils/processing_utils.py @@ -169,7 +169,14 @@ def refine_product_headers(product, total_obj_list): # Re-format ACS filter specification if phdu['instrume'] == 'ACS': phdu['filter'] = get_acs_filters(hdu, delimiter=';') - + + # WFPC2 update aperture if in poller file + if (total_obj_list[0].aperture_from_poller != 'empty_aperture') & (phdu['instrume'] =='WFPC2'): + log.info(f"Updating aperture header keyword from {phdu['aperture']} to {total_obj_list[0].aperture_from_poller}") + phdu['aperture'] = total_obj_list[0].aperture_from_poller + else: + log.debug("Not updating aperture keyword.") + # Insure PHOT* keywords are always in SCI extension for pkw in PHOT_KEYWORDS: if pkw in phdu: diff --git a/drizzlepac/haputils/product.py b/drizzlepac/haputils/product.py index 896b75009..3dc4b6ba2 100755 --- a/drizzlepac/haputils/product.py +++ b/drizzlepac/haputils/product.py @@ -64,20 +64,24 @@ class HAPProduct: """ def __init__( - self, prop_id, obset_id, instrument, detector, filename, filetype, log_level + self, prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ): # set logging level to user-specified level log.setLevel(log_level) self.log_level = log_level + # Special logic to specify the WFPC2 detector name always to PC (as opposed to WFPC2) for filenaming. + if instrument =='wfpc2': + detector = 'pc' + # Make sure the proposal ID is a 5-character string self.prop_id = prop_id.zfill(5) self.obset_id = obset_id self.instrument = instrument self.detector = detector + self.aperture_from_poller = aperture_from_poller self.filetype = filetype self.rules_file = None - self.basename = ( "hst_" + "_".join(map(str, [prop_id, obset_id, instrument, detector])) + "_" ) @@ -533,13 +537,13 @@ class TotalProduct(HAPProduct): """ def __init__( - self, prop_id, obset_id, instrument, detector, filename, filetype, log_level + self, prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, filename, filetype] + [prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype] ) self.exposure_name = filename[0:6] @@ -699,17 +703,18 @@ def __init__( obset_id, instrument, detector, + aperture_from_poller, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filters, filetype] ) if filename[0:7].lower() != "metawcs": self.exposure_name = filename[0:6] @@ -846,17 +851,18 @@ def __init__( obset_id, instrument, detector, + aperture_from_poller, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filters, filetype] ) self.filters = filters self.full_filename = self.copy_exposure(filename) @@ -1011,17 +1017,18 @@ def __init__( obset_id, instrument, detector, + aperture_from_poller, filename, filters, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) self.info = "_".join( - [prop_id, obset_id, instrument, detector, filename, filters, filetype] + [prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filters, filetype] ) self.filters = filters self.full_filename = self.copy_exposure(filename) @@ -1107,13 +1114,14 @@ def __init__( obset_id, instrument, detector, + aperture_from_poller, filename, layer, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, filename, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, filename, filetype, log_level ) filter_str = layer[0] @@ -1342,13 +1350,14 @@ def __init__( obset_id, instrument, detector, + aperture_from_poller, skycell_name, layer, filetype, log_level, ): super().__init__( - prop_id, obset_id, instrument, detector, skycell_name, filetype, log_level + prop_id, obset_id, instrument, detector, aperture_from_poller, skycell_name, filetype, log_level ) # May need to exclude 'filter' component from layer_str filter_str = layer[0] @@ -1366,7 +1375,7 @@ def __init__( layer_scale = layer[1] self.info = "_".join( - ["hst", skycell_name, instrument, detector, filter_str, layer_str] + ["hst", skycell_name, instrument, detector, aperture_from_poller, filter_str, layer_str] ) self.exposure_name = skycell_name self.cell_id = skycell_name.strip("skycell-") diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_alignment_all.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_alignment_all.json similarity index 100% rename from drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_alignment_all.json rename to drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_alignment_all.json diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json similarity index 100% rename from drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json rename to drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n2.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n2.json similarity index 100% rename from drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n2.json rename to drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n2.json diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n4.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n4.json similarity index 100% rename from drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n4.json rename to drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n4.json diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_total.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_total.json similarity index 100% rename from drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_total.json rename to drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_total.json diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_catalog_generation_all.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_catalog_generation_all.json similarity index 100% rename from drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_catalog_generation_all.json rename to drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_catalog_generation_all.json diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_quality_control_all.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_quality_control_all.json similarity index 100% rename from drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_quality_control_all.json rename to drizzlepac/pars/hap_pars/mvm_parameters/wfpc2/pc/wfpc2_pc_quality_control_all.json diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2_pc_index.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2_pc_index.json new file mode 100644 index 000000000..0aa496146 --- /dev/null +++ b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2_pc_index.json @@ -0,0 +1,19 @@ +{ + "alignment": { + "all": "wfpc2/pc/wfpc2_pc_alignment_all.json" + }, + "astrodrizzle": { + "any_n1": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json", + "filter_basic": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_total.json", + "single_basic": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json", + "wfpc2_pc_any_n2": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_n2.json", + "wfpc2_pc_any_n4": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_n4.json", + "wfpc2_pc_any_total": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_total.json" + }, + "catalog generation": { + "all": "wfpc2/pc/wfpc2_pc_catalog_generation_all.json" + }, + "quality control": { + "all": "wfpc2/pc/wfpc2_pc_quality_control_all.json" + } +} \ No newline at end of file diff --git a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2_wfpc2_index.json b/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2_wfpc2_index.json deleted file mode 100644 index e4b99009c..000000000 --- a/drizzlepac/pars/hap_pars/mvm_parameters/wfpc2_wfpc2_index.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "alignment": { - "all": "wfpc2/wfpc2/wfpc2_wfpc2_alignment_all.json" - }, - "astrodrizzle": { - "any_n1": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json", - "filter_basic": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_total.json", - "single_basic": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json", - "wfpc2_wfpc2_any_n2": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n2.json", - "wfpc2_wfpc2_any_n4": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n4.json", - "wfpc2_wfpc2_any_total": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_total.json" - }, - "catalog generation": { - "all": "wfpc2/wfpc2/wfpc2_wfpc2_catalog_generation_all.json" - }, - "quality control": { - "all": "wfpc2/wfpc2/wfpc2_wfpc2_quality_control_all.json" - } -} \ No newline at end of file diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_alignment_all.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_alignment_all.json similarity index 100% rename from drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_alignment_all.json rename to drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_alignment_all.json diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json similarity index 99% rename from drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json rename to drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json index 543ee3379..bc454bc97 100644 --- a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json +++ b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json @@ -73,7 +73,7 @@ "driz_combine": true, "final_pixfrac": 1.0, "final_fillval": null, - "final_bits": "65535", + "final_bits": "65533", "final_maskval": null, "final_wht_type": "EXP", "final_kernel": "square", diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n2.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n2.json similarity index 99% rename from drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n2.json rename to drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n2.json index 182445757..45d099bff 100644 --- a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n2.json +++ b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n2.json @@ -73,7 +73,7 @@ "driz_combine": true, "final_pixfrac": 1.0, "final_fillval": null, - "final_bits": "61439", + "final_bits": "61437", "final_maskval": null, "final_wht_type": "EXP", "final_kernel": "square", diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n3.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n3.json similarity index 100% rename from drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n3.json rename to drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n3.json diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n4.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n4.json similarity index 100% rename from drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n4.json rename to drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_n4.json diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_total.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_total.json similarity index 100% rename from drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_total.json rename to drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_astrodrizzle_any_total.json diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_catalog_generation_all.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_catalog_generation_all.json similarity index 100% rename from drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_catalog_generation_all.json rename to drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_catalog_generation_all.json diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_quality_control_all.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_quality_control_all.json similarity index 100% rename from drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_quality_control_all.json rename to drizzlepac/pars/hap_pars/svm_parameters/wfpc2/pc/wfpc2_pc_quality_control_all.json diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2_pc_index.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2_pc_index.json new file mode 100644 index 000000000..d5fa7e733 --- /dev/null +++ b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2_pc_index.json @@ -0,0 +1,20 @@ +{ + "alignment": { + "all": "wfpc2/pc/wfpc2_pc_alignment_all.json" + }, + "astrodrizzle": { + "any_n1": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json", + "filter_basic": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_total.json", + "single_basic": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_n1.json", + "total_basic": "wfpc2/pc/wfpc2_pc_astrodrizzle_any_total.json", + "wfpc2_pc_any_n2":"wfpc2/pc/wfpc2_pc_astrodrizzle_any_n2.json", + "wfpc2_pc_any_n3":"wfpc2/pc/wfpc2_pc_astrodrizzle_any_n3.json", + "wfpc2_pc_any_n4":"wfpc2/pc/wfpc2_pc_astrodrizzle_any_n4.json" + }, + "catalog generation": { + "all": "wfpc2/pc/wfpc2_pc_catalog_generation_all.json" + }, + "quality control": { + "all": "wfpc2/pc/wfpc2_pc_quality_control_all.json" + } +} \ No newline at end of file diff --git a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2_wfpc2_index.json b/drizzlepac/pars/hap_pars/svm_parameters/wfpc2_wfpc2_index.json deleted file mode 100644 index 3008d5708..000000000 --- a/drizzlepac/pars/hap_pars/svm_parameters/wfpc2_wfpc2_index.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "alignment": { - "all": "wfpc2/wfpc2/wfpc2_wfpc2_alignment_all.json" - }, - "astrodrizzle": { - "any_n1": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json", - "filter_basic": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_total.json", - "single_basic": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n1.json", - "total_basic": "wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_total.json", - "wfpc2_wfpc2_any_n2":"wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n2.json", - "wfpc2_wfpc2_any_n3":"wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n3.json", - "wfpc2_wfpc2_any_n4":"wfpc2/wfpc2/wfpc2_wfpc2_astrodrizzle_any_n4.json" - }, - "catalog generation": { - "all": "wfpc2/wfpc2/wfpc2_wfpc2_catalog_generation_all.json" - }, - "quality control": { - "all": "wfpc2/wfpc2/wfpc2_wfpc2_quality_control_all.json" - } -} \ No newline at end of file diff --git a/drizzlepac/runastrodriz.py b/drizzlepac/runastrodriz.py index 56c9a832d..5102e7a95 100755 --- a/drizzlepac/runastrodriz.py +++ b/drizzlepac/runastrodriz.py @@ -162,7 +162,7 @@ "ACS/WFC": {'sigma': 1.5, 'good_bits': 1360}, "ACS/SBC": {'sigma': 2.0, 'good_bits': 0}, "ACS/HRC": {'sigma': 1.5, 'good_bits': 1360}, - "WFPC2/WFPC2": {'sigma': 1.5, 'good_bits': 1360}} + "WFPC2/PC": {'sigma': 1.5, 'good_bits': 1360}} sub_dirs = ['OrIg_files', 'pipeline-default'] valid_alignment_modes = ['apriori', 'aposteriori', 'default-pipeline'] @@ -200,7 +200,7 @@ # Primary user interface def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, headerlets=True, align_to_gaia=True, force_alignment=False, - do_verify_guiding=True, debug=False, make_manifest=False): + do_verify_guiding=False, debug=False, make_manifest=False): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ @@ -277,7 +277,7 @@ def process(inFile, force=False, newpath=None, num_cores=None, inmemory=True, else: # start by turning off 'verify_guiding' since loss of lock # was exceedingly rare and noted in the quality keywords - # which get checked in '_anayze_exposure'. + # which get checked in '_analyze_exposure'. do_verify_guiding = False # Convert input c0m file into compatible flt file if 'd0m' in inFilename: diff --git a/drizzlepac/util.py b/drizzlepac/util.py index 55f7eb49f..3910a58c9 100644 --- a/drizzlepac/util.py +++ b/drizzlepac/util.py @@ -1174,7 +1174,7 @@ def base_taskname(taskname, packagename=None): ---------- taskname : str, None Full task name. If it is ``None``, :py:func:`base_taskname` will - return ``None``\ . + return ``None`` . packagename : str, None (Default = None) Package name. It is assumed that a compound task name is formed by diff --git a/drizzlepac/wfpc2Data.py b/drizzlepac/wfpc2Data.py index 82c031350..65a6fe50b 100644 --- a/drizzlepac/wfpc2Data.py +++ b/drizzlepac/wfpc2Data.py @@ -397,7 +397,7 @@ def wfpc2_to_flt(imgname): # Add keywords to be more compatible with ACS and WFC3 data num_sci = fileutil.countExtn(imgname) - det_name = 'WFPC2' + det_name = 'PC' in_sci[0].header['DETECTOR'] = det_name in_sci[0].header['PRIMESI'] = det_name diff --git a/requirements-dev.txt b/requirements-dev.txt index 8bf44230d..033b85872 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,9 @@ git+https://github.com/astropy/photutils.git#egg=photutils git+https://github.com/spacetelescope/stsci.tools.git#egg=stsci.tools git+https://github.com/spacetelescope/stwcs.git#egg=stwcs -git+https://github.com/astropy/astroquery.git#egg=astroquery +#git+https://github.com/astropy/astroquery.git#egg=astroquery --extra-index-url https://pypi.anaconda.org/astropy/simple astropy --pre + +--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy>=0.0.dev0 +scipy>=0.0.dev0