Skip to content

Commit

Permalink
Merge pull request #29 from simonsobs/stitching
Browse files Browse the repository at this point in the history
Stitching
  • Loading branch information
mattyowl authored Jul 24, 2020
2 parents af96760 + b3bd5f9 commit 32555c3
Show file tree
Hide file tree
Showing 7 changed files with 358 additions and 61 deletions.
38 changes: 21 additions & 17 deletions bin/nemo
Original file line number Diff line number Diff line change
Expand Up @@ -89,23 +89,6 @@ if __name__ == '__main__':
addInfo = addInfo, color = "cyan")
else:
if config.rank == 0: print("... already made catalog %s ..." % (optimalCatalogFileName))

# Stitch together map tiles - these are 'quicklook' images (downsampled by factor 4 in resolution)
# The stitchTiles routine will only write output if there are multiple maps matching the file pattern
if config.rank == 0 and 'makeQuickLookMaps' in config.parDict.keys() and config.parDict['makeQuickLookMaps'] == True:
# RMS (we don't save quicklook to selFnDir - users should use full-fat maps to be sure)
if os.path.exists(config.diagnosticsDir+os.path.sep+"quicklook_RMSMap.fits") == False:
maps.stitchTiles(config.selFnDir+os.path.sep+"RMSMap*.fits",
config.diagnosticsDir+os.path.sep+"quicklook_RMSMap.fits",
config.quicklookWCS, config.quicklookShape, fluxRescale = config.quicklookScale)
else:
print("... already made %s ..." % (config.diagnosticsDir+os.path.sep+"quicklook_RMSMap.fits"))
# S/N maps at reference filter scale
quicklookSNMapPath=config.filteredMapsDir+os.path.sep+"quicklook_%s_SNMap.fits" % (config.parDict['photFilter'])
if config.parDict['photFilter'] is not None and os.path.exists(quicklookSNMapPath) == False:
maps.stitchTiles(config.filteredMapsDir+os.path.sep+"*"+os.path.sep+"%s*SNMap.fits" % (config.parDict['photFilter']),
quicklookSNMapPath, config.quicklookWCS, config.quicklookShape,
fluxRescale = config.quicklookScale)

# Q function (filter mismatch) - if needed options have been given
# We may as well do this here to save having to run nemoMass separately (though we still can...)
Expand Down Expand Up @@ -185,6 +168,27 @@ if __name__ == '__main__':

# Tidying up etc.
if config.rank == 0:

# Stitch together map tiles - full fat versions (this will only work 'saveFilteredMaps: True')
if 'stitchTiles' in config.parDict.keys() and config.parDict['stitchTiles'] == True:
maps.stitchTiles(config)

# Stitch together map tiles - 'quicklook' images (downsampled by factor 4 in resolution)
# The stitchTilesQuickLook routine will only write output if there are multiple maps matching the file pattern
if 'makeQuickLookMaps' in config.parDict.keys() and config.parDict['makeQuickLookMaps'] == True:
# RMS (we don't save quicklook to selFnDir - users should use full-fat maps to be sure)
if os.path.exists(config.diagnosticsDir+os.path.sep+"quicklook_RMSMap.fits") == False:
maps.stitchTilesQuickLook(config.selFnDir+os.path.sep+"RMSMap*.fits",
config.diagnosticsDir+os.path.sep+"quicklook_RMSMap.fits",
config.quicklookWCS, config.quicklookShape, fluxRescale = config.quicklookScale)
else:
print("... already made %s ..." % (config.diagnosticsDir+os.path.sep+"quicklook_RMSMap.fits"))
# S/N maps at reference filter scale
quicklookSNMapPath=config.filteredMapsDir+os.path.sep+"quicklook_%s_SNMap.fits" % (config.parDict['photFilter'])
if config.parDict['photFilter'] is not None and os.path.exists(quicklookSNMapPath) == False:
maps.stitchTilesQuickLook(config.filteredMapsDir+os.path.sep+"*"+os.path.sep+"%s*SNMap.fits" % (config.parDict['photFilter']),
quicklookSNMapPath, config.quicklookWCS, config.quicklookShape,
fluxRescale = config.quicklookScale)

# Plot tile-averaged position recovery test
if sourceInjTable is not None:
Expand Down
9 changes: 9 additions & 0 deletions examples/pointSources/PS_S18d_f220_202006.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/bin/sh
#SBATCH --nodes=7
#SBATCH --ntasks-per-node=20
#SBATCH --mem=64000
#SBATCH --time=23:59:00

source ~/.bashrc
time mpiexec nemo PS_S18d_f220_202006.yml -M -n

177 changes: 177 additions & 0 deletions examples/pointSources/PS_S18d_f220_202006.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
# Nemo config file
# YAML format
# - use null to return None in Python
# - note that YAML is fussy about large numbers: use e.g. 1.0e+14 for M500MSun (not 1e14)

# Valid units are uK or Jy/sr
# this should be a list of maps at different frequencies
# NOTE: surveyMask is optional
unfilteredMaps:
- {mapFileName: "maps/Jun2020/act_s08_s18_cmb_f220_daynight_map.fits",
weightsFileName: "maps/Jun2020/act_s08_s18_cmb_f220_daynight_ivar.fits",
obsFreqGHz: 220.0, units: 'uK',
beamFileName: "maps/Jun2020/beams/s17_pa4_f220_nohwp_night_beam_profile_jitter.txt"}
# - {mapFileName: "maps/Mar2020/act_s08_s18_cmb_f090_daynight_map.fits",
# weightsFileName: "maps/Mar2020/act_s08_s18_cmb_f090_daynight_ivar.fits",
# obsFreqGHz: 98.3, units: 'uK',
# beamFileName: "Beams/190809/b20190809_s16_pa3_f090_nohwp_night_beam_profile_jitter_cmb.txt"}

# Masks
surveyMask: "maps/Jun2020/AdvACTSurveyMask_v7_S18.fits"
#surveyMask: 'maps/Sep2019/surveyMask_v7_S18_inc_extsrc.fits'

# Instead of giving point source mask, can give catalog instead
#maskPointSourcesFromCatalog:
# - "PS_S18_f150_auto/PS_S18_f150_auto_optimalCatalog.fits"
# - "customPSMask_S18/customPSCatalog_S18.fits"

# Detection/catalog options
# Set useInterpolator; True for sub-pixel flux and SNR measurements
thresholdSigma: 4.0
minObjPix: 1
findCenterOfMass: True
useInterpolator: True
rejectBorder: 0
objIdent: 'ACT-S'
longNames: False

# Photometry options
#photFilter: 'Arnaud_M2e14_z0p4'

# Optionally override the GNFW parameters - if not present, Arnaud et al. (2010) parameters are used
# The example below is for the Planck Pressure Profile (PPP)
#GNFWParams: {P0: 6.41, c500: 1.81, gamma: 0.31, alpha: 1.33, beta: 4.13, tol: 1e-7, npts: 100}

# Mass measurement options - used by nemoMass and nemoSelFn scripts
# Writes out .fits file to nemoOutDir/nemoOutDir_M500.fits
# redshiftCatalog: A .fits table containing name, RADeg, decDeg, redshift, redshiftErr columns
# forcedPhotometry: If True, calc mass based on extracted y0~ in 'photFilter' map at RADeg, decDeg as given in redshiftCatalog
# If False, cross match redshiftCatalog with optimal catalog made by nemo
# Q: If 'H13', use fit to Q from results presented in H13
# If 'fit', use fit to (theta, Q) done by nemo for 'photFilter' kernel
# tenToA0, B0, Mpivot, sigma_int: Fixed scaling relation options (see H13 or ACTPol paper)
# rescaleFactor, rescaleFactorErr: For MCal masses, as in the ACTPol paper (i.e., just rescales M500 results by 1/rescaleFactor)
#massOptions: {tenToA0: 4.95e-5,
# B0: 0.08,
# Mpivot: 3.0e+14,
# sigma_int: 0.2,
# relativisticCorrection: True,
# rescaleFactor: 0.68,
# rescaleFactorErr: 0.11,
# redshiftCatalog: "AdvACT_redshifts.fits",
# forcedPhotometry: False,
# Q: 'fit'}

# Selection function options
# NOTE: could eventually add 'completenessFraction' to 'massLimitMaps', which is why that's a dictionary list
# Use selFnFootprints to calculate average completeness in given sky areas - e.g., overlap with optical surveys
#calcSelFn: True
#selFnOptions: {fixedSNRCut: 5.0,
# massLimitMaps: [{z: 0.5}]}

#selFnFootprints:
# - {label: "HSC",
# maskList: ["HSCCheckAndSelFn/s19a_fdfc_CAR_contarea_ziy-gt-5.fits"]}
# - {label: "KiDS",
# maskList: ["KiDSSelFn/mask_KiDSN.fits", "KiDSSelFn/mask_KiDSS.fits"]}
# - {label: "DES",
# maskList: ["DESY3/AdvACT_y3a2_footprint_griz_1exp_v2.0.fits"]}

# Filter definitions:
# allFilters is a dictionary of parameters that will be copied into all mapFilters
# (these can be overridden by keys with the same name in mapFilters)
#allFilters: {class: "ArnaudModelMatchedFilter",
# params: {noiseParams: {method: "dataMap",
# noiseGridArcmin: 40.},
# saveFilteredMaps: False,
# saveRMSMap: False,
# savePlots: False,
# saveDS9Regions: False,
# outputUnits: 'yc',
# edgeTrimArcmin: 0.0}
# }

# mapFilters is a list of all the different filters to apply
# (keys in mapFilters with the same name as those in allFilters take priority)
#mapFilters:
# - {label: "Arnaud_M1e14_z0p2",
# params: {M500MSun: 1.0e+14, z: 0.2}}
# - {label: "Arnaud_M2e14_z0p2",
# params: {M500MSun: 2.0e+14, z: 0.2}}
# - {label: "Arnaud_M4e14_z0p2",
# params: {M500MSun: 4.0e+14, z: 0.2}}
# - {label: "Arnaud_M8e14_z0p2",
# params: {M500MSun: 8.0e+14, z: 0.2}}
# - {label: "Arnaud_M1e14_z0p4",
# params: {M500MSun: 1.0e+14, z: 0.4}}
# - {label: "Arnaud_M2e14_z0p4",
# params: {M500MSun: 2.0e+14, z: 0.4,
# saveFilteredMaps: True,
# savePlots: True}}
# - {label: "Arnaud_M4e14_z0p4",
# params: {M500MSun: 4.0e+14, z: 0.4}}
# - {label: "Arnaud_M8e14_z0p4",
# params: {M500MSun: 8.0e+14, z: 0.4}}
# - {label: "Arnaud_M1e14_z0p8",
# params: {M500MSun: 1.0e+14, z: 0.8}}
# - {label: "Arnaud_M2e14_z0p8",
# params: {M500MSun: 2.0e+14, z: 0.8}}
# - {label: "Arnaud_M4e14_z0p8",
# params: {M500MSun: 4.0e+14, z: 0.8}}
# - {label: "Arnaud_M8e14_z0p8",
# params: {M500MSun: 8.0e+14, z: 0.8}}
# - {label: "Arnaud_M1e14_z1p2",
# params: {M500MSun: 1.0e+14, z: 1.2}}
# - {label: "Arnaud_M2e14_z1p2",
# params: {M500MSun: 2.0e+14, z: 1.2}}
# - {label: "Arnaud_M4e14_z1p2",
# params: {M500MSun: 4.0e+14, z: 1.2}}
# - {label: "Arnaud_M8e14_z1p2",
# params: {M500MSun: 8.0e+14, z: 1.2}}

# Set this to True to generate a sky sim (with noise), run all the filters over it, and measure contamination
# Set numSkySims to number required - we need to average over many as results vary a fair bit
estimateContaminationFromSkySim: False
numSkySims: 10

# Set this to True to estimate contamination by running cluster finder over inverted maps
# This is sensitive to how well point source masking is done
estimateContaminationFromInvertedMaps: False

# Run position recovery test
positionRecoveryTest: False
posRecIterations: 1
posRecSourcesPerTile: 200
posRecModels:
- {redshift: 0.8, M500: 2.0e+14}
- {redshift: 0.4, M500: 2.0e+14}
- {redshift: 0.2, M500: 2.0e+14}
- {redshift: 0.1, M500: 2.0e+14}

# tileDir options - cut-up each map into smaller sections
makeTileDir: True
makeQuickLookMaps: True
tileOverlapDeg: 1.0
tileDefLabel: 'auto'
tileDefinitions: {mask: 'maps/Jun2020/AdvACTSurveyMask_v7_S18.fits',
targetTileWidthDeg: 10.0,
targetTileHeightDeg: 5.0}

# Filter definitions:
mapFilters:
- {label: "Beam",
class: "BeamMatchedFilter",
params: {noiseParams: {method: "model",
noiseGridArcmin: "smart",
numNoiseBins: 100},
saveFilteredMaps: True,
outputUnits: 'uK',
edgeTrimArcmin: 0.0}}

# If this is given, only the named tiles will be processed (useful for testing)
#tileNameList:
#- '1_10_7' # powerful f150 source; do as set - sensitive to point-source mask threshold
#- '1_10_8' # J2327 (next to a source); do as set - sensitive to point-source mask threshold
#- '2_0_7' # powerful f150 source
#- '2_2_8' # powerful f150 source
#- '3_0_1' # powerful f150 source
8 changes: 4 additions & 4 deletions nemo/completeness.py
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,7 @@ def getRMSTab(tileName, photFilterLabel, selFnDir, diagnosticsDir = None, footpr
RMSTab.add_column(atpy.Column(tileArea, 'areaDeg2'))
RMSTab.add_column(atpy.Column(RMSValues, 'y0RMS'))
# Sanity checks - these should be impossible but we have seen (e.g., when messed up masks)
tol=1e-3
tol=0.003
if abs(RMSTab['areaDeg2'].sum()-areaMapSqDeg.sum()) > tol:
raise Exception("Mismatch between area map and area in RMSTab for tile '%s'" % (tileName))
if np.less(RMSTab['areaDeg2'], 0).sum() > 0:
Expand Down Expand Up @@ -1119,9 +1119,9 @@ def makeFullSurveyMassLimitMapPlot(z, config):
config.quicklookShape, config.quicklookWCS=maps.shrinkWCS(config.origShape, config.origWCS, config.quicklookScale)

outFileName=config.diagnosticsDir+os.path.sep+"reproj_massLimitMap_z%s.fits" % (str(z).replace(".", "p"))
maps.stitchTiles(config.diagnosticsDir+os.path.sep+"*"+os.path.sep+"massLimitMap_z%s#*.fits" % (str(z).replace(".", "p")),
outFileName, config.quicklookWCS, config.quicklookShape,
fluxRescale = config.quicklookScale)
maps.stitchTilesQuickLook(config.diagnosticsDir+os.path.sep+"*"+os.path.sep+"massLimitMap_z%s#*.fits" % (str(z).replace(".", "p")),
outFileName, config.quicklookWCS, config.quicklookShape,
fluxRescale = config.quicklookScale)

# Make plot
if os.path.exists(outFileName) == True:
Expand Down
Loading

0 comments on commit 32555c3

Please sign in to comment.